aisquared.config.ModelConfiguration

  1from aisquared.base import BaseObject, CustomObject, ALLOWED_STAGES
  2from aisquared.config.harvesting import ImageHarvester, TextHarvester, InputHarvester
  3from aisquared.config.preprocessing.tabular import TabularPreprocessor
  4from aisquared.config.preprocessing.image import ImagePreprocessor
  5from aisquared.config.preprocessing.text import TextPreprocessor
  6from aisquared.config.analytic import DeployedAnalytic, DeployedModel, LocalModel, LocalAnalytic, ReverseMLWorkflow
  7from aisquared.config.postprocessing import BinaryClassification, MulticlassClassification, ObjectDetection, Regression
  8from aisquared.config.rendering import ImageRendering, ObjectRendering, DocumentRendering, WordRendering, FilterRendering
  9from aisquared.config.feedback import SimpleFeedback, BinaryFeedback, MulticlassFeedback, RegressionFeedback, ModelFeedback, QualitativeFeedback
 10
 11import tensorflowjs as tfjs
 12import tensorflow as tf
 13import shutil
 14import json
 15import os
 16
 17HARVESTING_CLASSES = (
 18    ImageHarvester,
 19    TextHarvester,
 20    InputHarvester,
 21    CustomObject
 22)
 23
 24PREPROCESSING_CLASSES = (
 25    TabularPreprocessor,
 26    ImagePreprocessor,
 27    TextPreprocessor,
 28    CustomObject
 29)
 30
 31ANALYTIC_CLASSES = (
 32    DeployedAnalytic,
 33    DeployedModel,
 34    LocalModel,
 35    LocalAnalytic,
 36    ReverseMLWorkflow,
 37    CustomObject
 38)
 39
 40POSTPROCESSING_CLASSES = (
 41    BinaryClassification,
 42    MulticlassClassification,
 43    ObjectDetection,
 44    Regression,
 45    CustomObject
 46)
 47
 48RENDERING_CLASSES = (
 49    ObjectRendering,
 50    ImageRendering,
 51    DocumentRendering,
 52    WordRendering,
 53    FilterRendering,
 54    CustomObject
 55)
 56
 57FEEDBACK_CLASSES = (
 58    ModelFeedback,
 59    SimpleFeedback,
 60    BinaryFeedback,
 61    MulticlassFeedback,
 62    RegressionFeedback,
 63    QualitativeFeedback
 64)
 65
 66LOCAL_CLASSES = (
 67    LocalModel,
 68    LocalAnalytic,
 69    CustomObject
 70)
 71
 72
 73class ModelConfiguration(BaseObject):
 74    """
 75    Configuration object for deploying a model or analytic
 76    """
 77
 78    def __init__(
 79            self,
 80            name,
 81            harvesting_steps,
 82            preprocessing_steps,
 83            analytic,
 84            postprocessing_steps,
 85            rendering_steps,
 86            feedback_steps=None,
 87            stage=ALLOWED_STAGES[0],
 88            version=None,
 89            description='',
 90            mlflow_uri=None,
 91            mlflow_user=None,
 92            mlflow_token=None,
 93            owner=None,
 94            url='*',
 95            auto_run=False
 96    ):
 97        """
 98        Parameters
 99        ----------
100        name : str
101            The name of the deployed analytic
102        harvesting_steps : None, Harvesting object or list of Harvesting objects
103            Harvesters to use with the analytic
104        preprocessing_steps : Preprocessing object or list of Preprocessing objects or None
105            Preprocessers to use
106        analytic : Analytic object or list of Analytic objects
107            Analytics to use
108        postprocessing_steps : Postprocessing object or list of Postprocessing objects or None
109            Postprocessers to use
110        rendering_steps : Rendering object or list of Rendering objects or None
111            Renderers to use
112        feedback_steps : None, Feedback object or list of Feedback objects or None (default None)
113            Feedback steps to use
114        stage : str (default 'experimental')
115            The stage of the model, from 'experimental', 'staging', 'production'
116        version : str or None (default None)
117            Version of the analytic
118        description : str (default '')
119            The description of the analytic
120        mlflow_uri : str or None (default None)
121            MLFlow URI to use, if applicable
122        mlflow_user : str or None (default None)
123            MLFlow user to use, if applicable
124        mlflow_token : str or None (default None)
125            MLFlow token to use, if applicable
126        owner : str or None (default None)
127            The owner of the model
128        url : str (default '*')
129            URL or URL pattern to match
130        auto_run : bool (default False)
131            Whether to automatically run this file when on a valid page
132        """
133        super().__init__()
134        self.name = name
135        self.harvesting_steps = harvesting_steps
136        self.preprocessing_steps = preprocessing_steps
137        self.analytic = analytic
138        self.postprocessing_steps = postprocessing_steps
139        self.rendering_steps = rendering_steps
140        self.stage = stage
141        self.feedback_steps = feedback_steps
142        self.version = version
143        self.description = description
144        self.mlflow_uri = mlflow_uri
145        self.mlflow_user = mlflow_user
146        self.mlflow_token = mlflow_token
147        self.owner = owner
148        self.url = url
149        self.auto_run = auto_run
150
151    # name
152    @property
153    def name(self):
154        return self._name
155
156    @name.setter
157    def name(self, value):
158        self._name = str(value)
159
160    # harvesting_steps
161    @property
162    def harvesting_steps(self):
163        return self._harvesting_steps
164
165    @harvesting_steps.setter
166    def harvesting_steps(self, value):
167        harvesting_classes = HARVESTING_CLASSES + (ModelConfiguration,)
168        if value is None or (isinstance(value, list) and all([val is None for val in value])):
169            self._harvesting_steps = value
170        elif isinstance(value, harvesting_classes):
171            self._harvesting_steps = [value]
172        elif isinstance(value, list) and all([isinstance(val, harvesting_classes) for val in value]):
173            self._harvesting_steps = value
174        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, harvesting_classes) for val in value for v in val]):
175            self._harvesting_steps = value
176        else:
177            raise ValueError(
178                'harvesting_steps must be a None, single Harvester object, a list of Harvester objects, or a list of list of harvester objects')
179
180    # preprocessing_steps
181    @property
182    def preprocessing_steps(self):
183        return self._preprocessing_steps
184
185    @preprocessing_steps.setter
186    def preprocessing_steps(self, value):
187        if value is None:
188            self._preprocessing_steps = value
189        elif isinstance(value, PREPROCESSING_CLASSES):
190            self._preprocessing_steps = [value]
191        elif isinstance(value, list) and all([isinstance(val, PREPROCESSING_CLASSES) for val in value]):
192            self._preprocessing_steps = value
193        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, PREPROCESSING_CLASSES) for val in value for v in val]):
194            self._preprocessing_steps = value
195        elif value is None:
196            self._preprocessing_steps = value
197        else:
198            raise ValueError(
199                'preprocessing_steps must a single Preprocessor object, a list of Preprocessor objects, or a list of list of preprocessor objects')
200
201    # analytic
202    @property
203    def analytic(self):
204        return self._analytic
205
206    @analytic.setter
207    def analytic(self, value):
208        if isinstance(value, ANALYTIC_CLASSES):
209            self._analytic = [value]
210        elif isinstance(value, list) and all([isinstance(val, ANALYTIC_CLASSES) for val in value]):
211            self._analytic = value
212        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, ANALYTIC_CLASSES) for val in value for v in val]):
213            self._analytic = value
214        else:
215            raise ValueError(
216                'analytic must be a single Analytic object, a list of Analytic objects, or a list of list of Analtyic objects')
217
218    # postprocessing_steps
219    @property
220    def postprocessing_steps(self):
221        return self._postprocessing_steps
222
223    @postprocessing_steps.setter
224    def postprocessing_steps(self, value):
225        if value is None:
226            self._postprocessing_steps = value
227        elif isinstance(value, POSTPROCESSING_CLASSES):
228            self._postprocessing_steps = [value]
229        elif isinstance(value, list) and all([isinstance(val, POSTPROCESSING_CLASSES) for val in value]):
230            self._postprocessing_steps = value
231        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, POSTPROCESSING_CLASSES) for val in value for v in val]):
232            self._postprocessing_steps = value
233        elif value is None:
234            self._postprocessing_steps = value
235        else:
236            raise ValueError(
237                'postprocessing_steps must be a single Postprocessing object, a list of Postprocessing objects, or a list of list of Postprocessing objects')
238
239    # rendering_steps
240    @property
241    def rendering_steps(self):
242        return self._rendering_steps
243
244    @rendering_steps.setter
245    def rendering_steps(self, value):
246        if isinstance(value, RENDERING_CLASSES) or value is None:
247            self._rendering_steps = [value]
248        elif isinstance(value, list) and all([isinstance(val, RENDERING_CLASSES) for val in value]):
249            self._rendering_steps = value
250        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, RENDERING_CLASSES) for val in value for v in val]):
251            self._rendering_steps = value
252        else:
253            raise ValueError(
254                'rendering_steps must be a single Rendering object, a list of Rendering objects, a list of list of Rendering objects, or None')
255
256    # feedback_steps
257    @property
258    def feedback_steps(self):
259        return self._feedback_steps
260
261    @feedback_steps.setter
262    def feedback_steps(self, value):
263        if value is None:
264            self._feedback_steps = value
265        elif isinstance(value, FEEDBACK_CLASSES):
266            self._feedback_steps = [value]
267        elif isinstance(value, list) and all([isinstance(val, FEEDBACK_CLASSES) for val in value]):
268            self._feedback_steps = value
269        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, FEEDBACK_CLASSES) for val in value for v in val]):
270            self._feedback_steps = value
271        else:
272            raise ValueError(
273                'feedback_steps must be a single Feedback object, a list of Feedback objects, a list of list of Feedback objects, or None')
274
275    # stage
276    @property
277    def stage(self):
278        return self._stage
279
280    @stage.setter
281    def stage(self, value):
282        if value not in ALLOWED_STAGES:
283            raise ValueError(f'stage must be one of {ALLOWED_STAGES}')
284        self._stage = value
285
286    # version
287    @property
288    def version(self):
289        return self._version
290
291    @version.setter
292    def version(self, value):
293        self._version = str(value) if value is not None else value
294
295    # description
296    @property
297    def description(self):
298        return self._description
299
300    @description.setter
301    def description(self, value):
302        self._description = str(value)
303
304    # mlflow_uri
305    @property
306    def mlflow_uri(self):
307        return self._mlflow_uri
308
309    @mlflow_uri.setter
310    def mlflow_uri(self, value):
311        self._mlflow_uri = value
312
313    # mlflow_user
314    @property
315    def mlflow_user(self):
316        return self._mlflow_user
317
318    @mlflow_user.setter
319    def mlflow_user(self, value):
320        self._mlflow_user = value
321
322    # mlflow_token
323    @property
324    def mlflow_token(self):
325        return self._mlflow_token
326
327    @mlflow_token.setter
328    def mlflow_token(self, value):
329        self._mlflow_token = value
330
331    # owner
332    @property
333    def owner(self):
334        return self._owner
335
336    @owner.setter
337    def owner(self, value):
338        if not isinstance(value, str) and value is not None:
339            raise ValueError('owner must be a string or None')
340        self._owner = value
341
342    # url
343    @property
344    def url(self):
345        return self._url
346
347    @url.setter
348    def url(self, value):
349        if not isinstance(value, str):
350            raise ValueError('url must be string')
351        self._url = value
352
353    # auto_run
354    @property
355    def auto_run(self):
356        return self._auto_run
357
358    @auto_run.setter
359    def auto_run(self, value):
360        if not isinstance(value, bool):
361            raise TypeError('auto_run must be Boolean valued')
362        self._auto_run = value
363
364    # harvester_dict
365    @property
366    def harvester_dict(self):
367        harvesting_classes = HARVESTING_CLASSES + (ModelConfiguration,)
368        if self.harvesting_steps is None or (isinstance(self.harvesting_steps, list) and all([val is None for val in self.harvesting_steps])):
369            return None
370        elif isinstance(self.harvesting_steps, list) and all([isinstance(val, harvesting_classes) for val in self.harvesting_steps]):
371            return [val.to_dict() for val in self.harvesting_steps]
372        else:
373            return [
374                [v.to_dict() for v in val] for val in self.harvesting_steps
375            ]
376
377    # preprocessing_dict
378    @property
379    def preprocesser_dict(self):
380        if self.preprocessing_steps is None:
381            return self.preprocessing_steps
382        elif isinstance(self.preprocessing_steps, list) and all([isinstance(val, PREPROCESSING_CLASSES) for val in self.preprocessing_steps]):
383            return [val.to_dict() for val in self.preprocessing_steps]
384        else:
385            return [
386                [v.to_dict() for v in val] for val in self.preprocessing_steps
387            ]
388
389    # analytic dict
390    @property
391    def analytic_dict(self):
392        if isinstance(self.analytic, list) and all([isinstance(val, ANALYTIC_CLASSES) for val in self.analytic]):
393            return [val.to_dict() for val in self.analytic]
394        else:
395            return [
396                [v.to_dict() for v in val] for val in self.analytic
397            ]
398
399    # postprocesser_dict
400    @property
401    def postprocesser_dict(self):
402        if self.postprocessing_steps is None:
403            return self.postprocessing_steps
404        elif isinstance(self.postprocessing_steps, list) and all([isinstance(val, POSTPROCESSING_CLASSES) for val in self.postprocessing_steps]):
405            return [val.to_dict() for val in self.postprocessing_steps]
406        else:
407            return [
408                [v.to_dict() for v in val] for val in self.postprocessing_steps
409            ]
410
411    # render_dict
412    @property
413    def render_dict(self):
414        if self.rendering_steps[0] is None:
415            return []
416        elif isinstance(self.rendering_steps, list) and all([isinstance(val, RENDERING_CLASSES) for val in self.rendering_steps]):
417            return [val.to_dict() for val in self.rendering_steps]
418        else:
419            return [
420                [v.to_dict() for v in val] for val in self.rendering_steps
421            ]
422
423    # feedback_dict
424    @property
425    def feedback_dict(self):
426        if self.feedback_steps is None:
427            return self.feedback_steps
428        elif isinstance(self.feedback_steps, list) and all([isinstance(val, FEEDBACK_CLASSES) for val in self.feedback_steps]):
429            return [val.to_dict() for val in self.feedback_steps]
430        else:
431            return [
432                [v.to_dict() for v in val] for val in self.feedback_steps
433            ]
434
435    def get_model_filenames(self):
436        """
437        Get filenames for all models in the configuration
438        """
439        filenames = []
440        if self.harvesting_steps is None or (isinstance(self.harvesting_steps, list) and all([val is None for val in self.harvesting_steps])):
441            harvesting_list = []
442        elif isinstance(self.harvesting_steps[0], list):
443            harvesting_list = [
444                h for harvester in self.harvesting_steps for h in harvester]
445        else:
446            harvesting_list = self.harvesting_steps
447        for harvester in harvesting_list:
448            if isinstance(harvester, ModelConfiguration):
449                filenames.extend(harvester.get_model_filenames())
450
451        if isinstance(self.analytic[0], ANALYTIC_CLASSES):
452            for a in self.analytic:
453                if isinstance(a, LOCAL_CLASSES):
454                    try:
455                        filenames.append(a.path)
456                    except Exception:
457                        pass
458        else:
459            for analytic in self.analytic:
460                for a in analytic:
461                    if isinstance(a, LOCAL_CLASSES):
462                        try:
463                            filenames.append(a.path)
464                        except Exception:
465                            pass
466        return [f for f in filenames if f is not None]
467
468    def to_dict(self):
469        """
470        Get the object as a dictionary
471        """
472        return {
473            'className': 'ModelConfiguration',
474            'params': {
475                'name': self.name,
476                'harvestingSteps': self.harvester_dict,
477                'preprocessingSteps': self.preprocesser_dict,
478                'analytics': self.analytic_dict,
479                'postprocessingSteps': self.postprocesser_dict,
480                'renderingSteps': self.render_dict,
481                'feedbackSteps': self.feedback_dict,
482                'stage': self.stage,
483                'version': self.version,
484                'description': self.description,
485                'mlflowUri': self.mlflow_uri,
486                'mlflowUser': self.mlflow_user,
487                'mlflowToken': self.mlflow_token,
488                'owner': self.owner,
489                'url': self.url,
490                'autoRun': self.auto_run
491            }
492        }
493
494    def compile(self, filename=None, dtype=None):
495        """
496        Compile the object into a '.air' file, which can then be dragged and
497        dropped into applications using the AI Squared JavaScript SDK
498
499        Parameters
500        ----------
501        filename : path-like or None (default None)
502            Filename to compile to. If None, defaults to '{NAME}.air', where {NAME} is the
503            name of the analytic
504        dtype : str or None (default None)
505            The datatype to use for the model weights. If None, defaults to 'float32'
506        """
507        if filename is None:
508            filename = self.name + '.air'
509
510        if dtype is None:
511            dtype_map = None
512        else:
513            dtype_map = {dtype: '*'}
514
515        dirname = os.path.join('.', os.path.splitext(filename)[0])
516
517        # write the object as json config
518        os.makedirs(dirname, exist_ok=True)
519        with open(os.path.join(dirname, 'config.json'), 'w') as f:
520            json.dump(self.to_dict(), f)
521
522        # go through the files and copy them/make them tfjs files
523        filenames = self.get_model_filenames()
524        if len(filenames) != 0:
525            for f in filenames:
526                if os.path.splitext(f)[-1] == '.h5':
527                    model = tf.keras.models.load_model(f)
528                    model_dir = os.path.join(dirname, os.path.split(f)[-1])
529                    tfjs.converters.save_keras_model(
530                        model, model_dir, quantization_dtype_map=dtype_map)
531                else:
532                    shutil.copy(f, dirname)
533
534        # go through the entire directory of dirname, grab all files, and make
535        # the archive file
536        shutil.make_archive(filename, 'zip', dirname)
537
538        # Move the archive file to just have .air
539        shutil.move(filename + '.zip', filename)
540
541        # Remove the temp directory
542        shutil.rmtree(dirname, ignore_errors=True)
class ModelConfiguration(aisquared.base.BaseObject.BaseObject):
 74class ModelConfiguration(BaseObject):
 75    """
 76    Configuration object for deploying a model or analytic
 77    """
 78
 79    def __init__(
 80            self,
 81            name,
 82            harvesting_steps,
 83            preprocessing_steps,
 84            analytic,
 85            postprocessing_steps,
 86            rendering_steps,
 87            feedback_steps=None,
 88            stage=ALLOWED_STAGES[0],
 89            version=None,
 90            description='',
 91            mlflow_uri=None,
 92            mlflow_user=None,
 93            mlflow_token=None,
 94            owner=None,
 95            url='*',
 96            auto_run=False
 97    ):
 98        """
 99        Parameters
100        ----------
101        name : str
102            The name of the deployed analytic
103        harvesting_steps : None, Harvesting object or list of Harvesting objects
104            Harvesters to use with the analytic
105        preprocessing_steps : Preprocessing object or list of Preprocessing objects or None
106            Preprocessers to use
107        analytic : Analytic object or list of Analytic objects
108            Analytics to use
109        postprocessing_steps : Postprocessing object or list of Postprocessing objects or None
110            Postprocessers to use
111        rendering_steps : Rendering object or list of Rendering objects or None
112            Renderers to use
113        feedback_steps : None, Feedback object or list of Feedback objects or None (default None)
114            Feedback steps to use
115        stage : str (default 'experimental')
116            The stage of the model, from 'experimental', 'staging', 'production'
117        version : str or None (default None)
118            Version of the analytic
119        description : str (default '')
120            The description of the analytic
121        mlflow_uri : str or None (default None)
122            MLFlow URI to use, if applicable
123        mlflow_user : str or None (default None)
124            MLFlow user to use, if applicable
125        mlflow_token : str or None (default None)
126            MLFlow token to use, if applicable
127        owner : str or None (default None)
128            The owner of the model
129        url : str (default '*')
130            URL or URL pattern to match
131        auto_run : bool (default False)
132            Whether to automatically run this file when on a valid page
133        """
134        super().__init__()
135        self.name = name
136        self.harvesting_steps = harvesting_steps
137        self.preprocessing_steps = preprocessing_steps
138        self.analytic = analytic
139        self.postprocessing_steps = postprocessing_steps
140        self.rendering_steps = rendering_steps
141        self.stage = stage
142        self.feedback_steps = feedback_steps
143        self.version = version
144        self.description = description
145        self.mlflow_uri = mlflow_uri
146        self.mlflow_user = mlflow_user
147        self.mlflow_token = mlflow_token
148        self.owner = owner
149        self.url = url
150        self.auto_run = auto_run
151
152    # name
153    @property
154    def name(self):
155        return self._name
156
157    @name.setter
158    def name(self, value):
159        self._name = str(value)
160
161    # harvesting_steps
162    @property
163    def harvesting_steps(self):
164        return self._harvesting_steps
165
166    @harvesting_steps.setter
167    def harvesting_steps(self, value):
168        harvesting_classes = HARVESTING_CLASSES + (ModelConfiguration,)
169        if value is None or (isinstance(value, list) and all([val is None for val in value])):
170            self._harvesting_steps = value
171        elif isinstance(value, harvesting_classes):
172            self._harvesting_steps = [value]
173        elif isinstance(value, list) and all([isinstance(val, harvesting_classes) for val in value]):
174            self._harvesting_steps = value
175        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, harvesting_classes) for val in value for v in val]):
176            self._harvesting_steps = value
177        else:
178            raise ValueError(
179                'harvesting_steps must be a None, single Harvester object, a list of Harvester objects, or a list of list of harvester objects')
180
181    # preprocessing_steps
182    @property
183    def preprocessing_steps(self):
184        return self._preprocessing_steps
185
186    @preprocessing_steps.setter
187    def preprocessing_steps(self, value):
188        if value is None:
189            self._preprocessing_steps = value
190        elif isinstance(value, PREPROCESSING_CLASSES):
191            self._preprocessing_steps = [value]
192        elif isinstance(value, list) and all([isinstance(val, PREPROCESSING_CLASSES) for val in value]):
193            self._preprocessing_steps = value
194        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, PREPROCESSING_CLASSES) for val in value for v in val]):
195            self._preprocessing_steps = value
196        elif value is None:
197            self._preprocessing_steps = value
198        else:
199            raise ValueError(
200                'preprocessing_steps must a single Preprocessor object, a list of Preprocessor objects, or a list of list of preprocessor objects')
201
202    # analytic
203    @property
204    def analytic(self):
205        return self._analytic
206
207    @analytic.setter
208    def analytic(self, value):
209        if isinstance(value, ANALYTIC_CLASSES):
210            self._analytic = [value]
211        elif isinstance(value, list) and all([isinstance(val, ANALYTIC_CLASSES) for val in value]):
212            self._analytic = value
213        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, ANALYTIC_CLASSES) for val in value for v in val]):
214            self._analytic = value
215        else:
216            raise ValueError(
217                'analytic must be a single Analytic object, a list of Analytic objects, or a list of list of Analtyic objects')
218
219    # postprocessing_steps
220    @property
221    def postprocessing_steps(self):
222        return self._postprocessing_steps
223
224    @postprocessing_steps.setter
225    def postprocessing_steps(self, value):
226        if value is None:
227            self._postprocessing_steps = value
228        elif isinstance(value, POSTPROCESSING_CLASSES):
229            self._postprocessing_steps = [value]
230        elif isinstance(value, list) and all([isinstance(val, POSTPROCESSING_CLASSES) for val in value]):
231            self._postprocessing_steps = value
232        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, POSTPROCESSING_CLASSES) for val in value for v in val]):
233            self._postprocessing_steps = value
234        elif value is None:
235            self._postprocessing_steps = value
236        else:
237            raise ValueError(
238                'postprocessing_steps must be a single Postprocessing object, a list of Postprocessing objects, or a list of list of Postprocessing objects')
239
240    # rendering_steps
241    @property
242    def rendering_steps(self):
243        return self._rendering_steps
244
245    @rendering_steps.setter
246    def rendering_steps(self, value):
247        if isinstance(value, RENDERING_CLASSES) or value is None:
248            self._rendering_steps = [value]
249        elif isinstance(value, list) and all([isinstance(val, RENDERING_CLASSES) for val in value]):
250            self._rendering_steps = value
251        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, RENDERING_CLASSES) for val in value for v in val]):
252            self._rendering_steps = value
253        else:
254            raise ValueError(
255                'rendering_steps must be a single Rendering object, a list of Rendering objects, a list of list of Rendering objects, or None')
256
257    # feedback_steps
258    @property
259    def feedback_steps(self):
260        return self._feedback_steps
261
262    @feedback_steps.setter
263    def feedback_steps(self, value):
264        if value is None:
265            self._feedback_steps = value
266        elif isinstance(value, FEEDBACK_CLASSES):
267            self._feedback_steps = [value]
268        elif isinstance(value, list) and all([isinstance(val, FEEDBACK_CLASSES) for val in value]):
269            self._feedback_steps = value
270        elif isinstance(value, list) and all([isinstance(val, list) for val in value]) and all([isinstance(v, FEEDBACK_CLASSES) for val in value for v in val]):
271            self._feedback_steps = value
272        else:
273            raise ValueError(
274                'feedback_steps must be a single Feedback object, a list of Feedback objects, a list of list of Feedback objects, or None')
275
276    # stage
277    @property
278    def stage(self):
279        return self._stage
280
281    @stage.setter
282    def stage(self, value):
283        if value not in ALLOWED_STAGES:
284            raise ValueError(f'stage must be one of {ALLOWED_STAGES}')
285        self._stage = value
286
287    # version
288    @property
289    def version(self):
290        return self._version
291
292    @version.setter
293    def version(self, value):
294        self._version = str(value) if value is not None else value
295
296    # description
297    @property
298    def description(self):
299        return self._description
300
301    @description.setter
302    def description(self, value):
303        self._description = str(value)
304
305    # mlflow_uri
306    @property
307    def mlflow_uri(self):
308        return self._mlflow_uri
309
310    @mlflow_uri.setter
311    def mlflow_uri(self, value):
312        self._mlflow_uri = value
313
314    # mlflow_user
315    @property
316    def mlflow_user(self):
317        return self._mlflow_user
318
319    @mlflow_user.setter
320    def mlflow_user(self, value):
321        self._mlflow_user = value
322
323    # mlflow_token
324    @property
325    def mlflow_token(self):
326        return self._mlflow_token
327
328    @mlflow_token.setter
329    def mlflow_token(self, value):
330        self._mlflow_token = value
331
332    # owner
333    @property
334    def owner(self):
335        return self._owner
336
337    @owner.setter
338    def owner(self, value):
339        if not isinstance(value, str) and value is not None:
340            raise ValueError('owner must be a string or None')
341        self._owner = value
342
343    # url
344    @property
345    def url(self):
346        return self._url
347
348    @url.setter
349    def url(self, value):
350        if not isinstance(value, str):
351            raise ValueError('url must be string')
352        self._url = value
353
354    # auto_run
355    @property
356    def auto_run(self):
357        return self._auto_run
358
359    @auto_run.setter
360    def auto_run(self, value):
361        if not isinstance(value, bool):
362            raise TypeError('auto_run must be Boolean valued')
363        self._auto_run = value
364
365    # harvester_dict
366    @property
367    def harvester_dict(self):
368        harvesting_classes = HARVESTING_CLASSES + (ModelConfiguration,)
369        if self.harvesting_steps is None or (isinstance(self.harvesting_steps, list) and all([val is None for val in self.harvesting_steps])):
370            return None
371        elif isinstance(self.harvesting_steps, list) and all([isinstance(val, harvesting_classes) for val in self.harvesting_steps]):
372            return [val.to_dict() for val in self.harvesting_steps]
373        else:
374            return [
375                [v.to_dict() for v in val] for val in self.harvesting_steps
376            ]
377
378    # preprocessing_dict
379    @property
380    def preprocesser_dict(self):
381        if self.preprocessing_steps is None:
382            return self.preprocessing_steps
383        elif isinstance(self.preprocessing_steps, list) and all([isinstance(val, PREPROCESSING_CLASSES) for val in self.preprocessing_steps]):
384            return [val.to_dict() for val in self.preprocessing_steps]
385        else:
386            return [
387                [v.to_dict() for v in val] for val in self.preprocessing_steps
388            ]
389
390    # analytic dict
391    @property
392    def analytic_dict(self):
393        if isinstance(self.analytic, list) and all([isinstance(val, ANALYTIC_CLASSES) for val in self.analytic]):
394            return [val.to_dict() for val in self.analytic]
395        else:
396            return [
397                [v.to_dict() for v in val] for val in self.analytic
398            ]
399
400    # postprocesser_dict
401    @property
402    def postprocesser_dict(self):
403        if self.postprocessing_steps is None:
404            return self.postprocessing_steps
405        elif isinstance(self.postprocessing_steps, list) and all([isinstance(val, POSTPROCESSING_CLASSES) for val in self.postprocessing_steps]):
406            return [val.to_dict() for val in self.postprocessing_steps]
407        else:
408            return [
409                [v.to_dict() for v in val] for val in self.postprocessing_steps
410            ]
411
412    # render_dict
413    @property
414    def render_dict(self):
415        if self.rendering_steps[0] is None:
416            return []
417        elif isinstance(self.rendering_steps, list) and all([isinstance(val, RENDERING_CLASSES) for val in self.rendering_steps]):
418            return [val.to_dict() for val in self.rendering_steps]
419        else:
420            return [
421                [v.to_dict() for v in val] for val in self.rendering_steps
422            ]
423
424    # feedback_dict
425    @property
426    def feedback_dict(self):
427        if self.feedback_steps is None:
428            return self.feedback_steps
429        elif isinstance(self.feedback_steps, list) and all([isinstance(val, FEEDBACK_CLASSES) for val in self.feedback_steps]):
430            return [val.to_dict() for val in self.feedback_steps]
431        else:
432            return [
433                [v.to_dict() for v in val] for val in self.feedback_steps
434            ]
435
436    def get_model_filenames(self):
437        """
438        Get filenames for all models in the configuration
439        """
440        filenames = []
441        if self.harvesting_steps is None or (isinstance(self.harvesting_steps, list) and all([val is None for val in self.harvesting_steps])):
442            harvesting_list = []
443        elif isinstance(self.harvesting_steps[0], list):
444            harvesting_list = [
445                h for harvester in self.harvesting_steps for h in harvester]
446        else:
447            harvesting_list = self.harvesting_steps
448        for harvester in harvesting_list:
449            if isinstance(harvester, ModelConfiguration):
450                filenames.extend(harvester.get_model_filenames())
451
452        if isinstance(self.analytic[0], ANALYTIC_CLASSES):
453            for a in self.analytic:
454                if isinstance(a, LOCAL_CLASSES):
455                    try:
456                        filenames.append(a.path)
457                    except Exception:
458                        pass
459        else:
460            for analytic in self.analytic:
461                for a in analytic:
462                    if isinstance(a, LOCAL_CLASSES):
463                        try:
464                            filenames.append(a.path)
465                        except Exception:
466                            pass
467        return [f for f in filenames if f is not None]
468
469    def to_dict(self):
470        """
471        Get the object as a dictionary
472        """
473        return {
474            'className': 'ModelConfiguration',
475            'params': {
476                'name': self.name,
477                'harvestingSteps': self.harvester_dict,
478                'preprocessingSteps': self.preprocesser_dict,
479                'analytics': self.analytic_dict,
480                'postprocessingSteps': self.postprocesser_dict,
481                'renderingSteps': self.render_dict,
482                'feedbackSteps': self.feedback_dict,
483                'stage': self.stage,
484                'version': self.version,
485                'description': self.description,
486                'mlflowUri': self.mlflow_uri,
487                'mlflowUser': self.mlflow_user,
488                'mlflowToken': self.mlflow_token,
489                'owner': self.owner,
490                'url': self.url,
491                'autoRun': self.auto_run
492            }
493        }
494
495    def compile(self, filename=None, dtype=None):
496        """
497        Compile the object into a '.air' file, which can then be dragged and
498        dropped into applications using the AI Squared JavaScript SDK
499
500        Parameters
501        ----------
502        filename : path-like or None (default None)
503            Filename to compile to. If None, defaults to '{NAME}.air', where {NAME} is the
504            name of the analytic
505        dtype : str or None (default None)
506            The datatype to use for the model weights. If None, defaults to 'float32'
507        """
508        if filename is None:
509            filename = self.name + '.air'
510
511        if dtype is None:
512            dtype_map = None
513        else:
514            dtype_map = {dtype: '*'}
515
516        dirname = os.path.join('.', os.path.splitext(filename)[0])
517
518        # write the object as json config
519        os.makedirs(dirname, exist_ok=True)
520        with open(os.path.join(dirname, 'config.json'), 'w') as f:
521            json.dump(self.to_dict(), f)
522
523        # go through the files and copy them/make them tfjs files
524        filenames = self.get_model_filenames()
525        if len(filenames) != 0:
526            for f in filenames:
527                if os.path.splitext(f)[-1] == '.h5':
528                    model = tf.keras.models.load_model(f)
529                    model_dir = os.path.join(dirname, os.path.split(f)[-1])
530                    tfjs.converters.save_keras_model(
531                        model, model_dir, quantization_dtype_map=dtype_map)
532                else:
533                    shutil.copy(f, dirname)
534
535        # go through the entire directory of dirname, grab all files, and make
536        # the archive file
537        shutil.make_archive(filename, 'zip', dirname)
538
539        # Move the archive file to just have .air
540        shutil.move(filename + '.zip', filename)
541
542        # Remove the temp directory
543        shutil.rmtree(dirname, ignore_errors=True)

Configuration object for deploying a model or analytic

ModelConfiguration( name, harvesting_steps, preprocessing_steps, analytic, postprocessing_steps, rendering_steps, feedback_steps=None, stage='experimental', version=None, description='', mlflow_uri=None, mlflow_user=None, mlflow_token=None, owner=None, url='*', auto_run=False)
 79    def __init__(
 80            self,
 81            name,
 82            harvesting_steps,
 83            preprocessing_steps,
 84            analytic,
 85            postprocessing_steps,
 86            rendering_steps,
 87            feedback_steps=None,
 88            stage=ALLOWED_STAGES[0],
 89            version=None,
 90            description='',
 91            mlflow_uri=None,
 92            mlflow_user=None,
 93            mlflow_token=None,
 94            owner=None,
 95            url='*',
 96            auto_run=False
 97    ):
 98        """
 99        Parameters
100        ----------
101        name : str
102            The name of the deployed analytic
103        harvesting_steps : None, Harvesting object or list of Harvesting objects
104            Harvesters to use with the analytic
105        preprocessing_steps : Preprocessing object or list of Preprocessing objects or None
106            Preprocessers to use
107        analytic : Analytic object or list of Analytic objects
108            Analytics to use
109        postprocessing_steps : Postprocessing object or list of Postprocessing objects or None
110            Postprocessers to use
111        rendering_steps : Rendering object or list of Rendering objects or None
112            Renderers to use
113        feedback_steps : None, Feedback object or list of Feedback objects or None (default None)
114            Feedback steps to use
115        stage : str (default 'experimental')
116            The stage of the model, from 'experimental', 'staging', 'production'
117        version : str or None (default None)
118            Version of the analytic
119        description : str (default '')
120            The description of the analytic
121        mlflow_uri : str or None (default None)
122            MLFlow URI to use, if applicable
123        mlflow_user : str or None (default None)
124            MLFlow user to use, if applicable
125        mlflow_token : str or None (default None)
126            MLFlow token to use, if applicable
127        owner : str or None (default None)
128            The owner of the model
129        url : str (default '*')
130            URL or URL pattern to match
131        auto_run : bool (default False)
132            Whether to automatically run this file when on a valid page
133        """
134        super().__init__()
135        self.name = name
136        self.harvesting_steps = harvesting_steps
137        self.preprocessing_steps = preprocessing_steps
138        self.analytic = analytic
139        self.postprocessing_steps = postprocessing_steps
140        self.rendering_steps = rendering_steps
141        self.stage = stage
142        self.feedback_steps = feedback_steps
143        self.version = version
144        self.description = description
145        self.mlflow_uri = mlflow_uri
146        self.mlflow_user = mlflow_user
147        self.mlflow_token = mlflow_token
148        self.owner = owner
149        self.url = url
150        self.auto_run = auto_run
Parameters
  • name (str): The name of the deployed analytic
  • harvesting_steps (None, Harvesting object or list of Harvesting objects): Harvesters to use with the analytic
  • preprocessing_steps (Preprocessing object or list of Preprocessing objects or None): Preprocessers to use
  • analytic (Analytic object or list of Analytic objects): Analytics to use
  • postprocessing_steps (Postprocessing object or list of Postprocessing objects or None): Postprocessers to use
  • rendering_steps (Rendering object or list of Rendering objects or None): Renderers to use
  • feedback_steps (None, Feedback object or list of Feedback objects or None (default None)): Feedback steps to use
  • stage (str (default 'experimental')): The stage of the model, from 'experimental', 'staging', 'production'
  • version (str or None (default None)): Version of the analytic
  • description (str (default '')): The description of the analytic
  • mlflow_uri (str or None (default None)): MLFlow URI to use, if applicable
  • mlflow_user (str or None (default None)): MLFlow user to use, if applicable
  • mlflow_token (str or None (default None)): MLFlow token to use, if applicable
  • owner (str or None (default None)): The owner of the model
  • url (str (default '*')): URL or URL pattern to match
  • auto_run (bool (default False)): Whether to automatically run this file when on a valid page
def get_model_filenames(self):
436    def get_model_filenames(self):
437        """
438        Get filenames for all models in the configuration
439        """
440        filenames = []
441        if self.harvesting_steps is None or (isinstance(self.harvesting_steps, list) and all([val is None for val in self.harvesting_steps])):
442            harvesting_list = []
443        elif isinstance(self.harvesting_steps[0], list):
444            harvesting_list = [
445                h for harvester in self.harvesting_steps for h in harvester]
446        else:
447            harvesting_list = self.harvesting_steps
448        for harvester in harvesting_list:
449            if isinstance(harvester, ModelConfiguration):
450                filenames.extend(harvester.get_model_filenames())
451
452        if isinstance(self.analytic[0], ANALYTIC_CLASSES):
453            for a in self.analytic:
454                if isinstance(a, LOCAL_CLASSES):
455                    try:
456                        filenames.append(a.path)
457                    except Exception:
458                        pass
459        else:
460            for analytic in self.analytic:
461                for a in analytic:
462                    if isinstance(a, LOCAL_CLASSES):
463                        try:
464                            filenames.append(a.path)
465                        except Exception:
466                            pass
467        return [f for f in filenames if f is not None]

Get filenames for all models in the configuration

def to_dict(self):
469    def to_dict(self):
470        """
471        Get the object as a dictionary
472        """
473        return {
474            'className': 'ModelConfiguration',
475            'params': {
476                'name': self.name,
477                'harvestingSteps': self.harvester_dict,
478                'preprocessingSteps': self.preprocesser_dict,
479                'analytics': self.analytic_dict,
480                'postprocessingSteps': self.postprocesser_dict,
481                'renderingSteps': self.render_dict,
482                'feedbackSteps': self.feedback_dict,
483                'stage': self.stage,
484                'version': self.version,
485                'description': self.description,
486                'mlflowUri': self.mlflow_uri,
487                'mlflowUser': self.mlflow_user,
488                'mlflowToken': self.mlflow_token,
489                'owner': self.owner,
490                'url': self.url,
491                'autoRun': self.auto_run
492            }
493        }

Get the object as a dictionary

def compile(self, filename=None, dtype=None):
495    def compile(self, filename=None, dtype=None):
496        """
497        Compile the object into a '.air' file, which can then be dragged and
498        dropped into applications using the AI Squared JavaScript SDK
499
500        Parameters
501        ----------
502        filename : path-like or None (default None)
503            Filename to compile to. If None, defaults to '{NAME}.air', where {NAME} is the
504            name of the analytic
505        dtype : str or None (default None)
506            The datatype to use for the model weights. If None, defaults to 'float32'
507        """
508        if filename is None:
509            filename = self.name + '.air'
510
511        if dtype is None:
512            dtype_map = None
513        else:
514            dtype_map = {dtype: '*'}
515
516        dirname = os.path.join('.', os.path.splitext(filename)[0])
517
518        # write the object as json config
519        os.makedirs(dirname, exist_ok=True)
520        with open(os.path.join(dirname, 'config.json'), 'w') as f:
521            json.dump(self.to_dict(), f)
522
523        # go through the files and copy them/make them tfjs files
524        filenames = self.get_model_filenames()
525        if len(filenames) != 0:
526            for f in filenames:
527                if os.path.splitext(f)[-1] == '.h5':
528                    model = tf.keras.models.load_model(f)
529                    model_dir = os.path.join(dirname, os.path.split(f)[-1])
530                    tfjs.converters.save_keras_model(
531                        model, model_dir, quantization_dtype_map=dtype_map)
532                else:
533                    shutil.copy(f, dirname)
534
535        # go through the entire directory of dirname, grab all files, and make
536        # the archive file
537        shutil.make_archive(filename, 'zip', dirname)
538
539        # Move the archive file to just have .air
540        shutil.move(filename + '.zip', filename)
541
542        # Remove the temp directory
543        shutil.rmtree(dirname, ignore_errors=True)

Compile the object into a '.air' file, which can then be dragged and dropped into applications using the AI Squared JavaScript SDK

Parameters
  • filename (path-like or None (default None)): Filename to compile to. If None, defaults to '{NAME}.air', where {NAME} is the name of the analytic
  • dtype (str or None (default None)): The datatype to use for the model weights. If None, defaults to 'float32'