setup.py 17.6 KB
Newer Older
1
2
3
4
from types import MethodType
import copy
import yaml
import os
Antoine Berchet's avatar
Antoine Berchet committed
5
6
7
import pycif.utils.check as check
import pycif.utils.dates as dates
from pycif.utils.check.errclass import PluginError
8
9

from .baseclass import Plugin
Antoine Berchet's avatar
Antoine Berchet committed
10
11
from pycif.utils.check import verbose
from pycif.utils.check import init_log, error, verbose
12

13
import pdb
14
15
16
17
18

class Setup(Plugin):
    
    @classmethod
    def run_simu(cls, args):
19
        
20
        config_dict = cls.from_yaml(args['def_file'])
21
        
22
23
24
25
26
27
28
29
30
31
        # Creates and initializes the log file
        config_dict['logfile'], config_dict['workdir'] \
            = init_log(config_dict['logfile'],
                       config_dict['workdir'],
                       config_dict['verbose'])
        cls.config_verbose(config_dict)

        # Copying Yaml file for traceability of simulations
        os.system('cp ' + args['def_file'] + ' ' + config_dict['workdir'] + '/')

Joel Thanwerdas's avatar
Joel Thanwerdas committed
32
33
34
        # Looking for others Yaml configs files in the main config_file
        config_dict = cls.yaml_subconfigs(config_dict)

35
36
        # Load a dictionary to a Setup recursive object
        setup = Setup.from_dict(config_dict)
37
        
38
39
40
41
        # Initialize every plugin, requirements and data
        cls.load_setup(setup, level=0)
        
        # Run the mode
42
43
44
45
46
47
            
        # Initialize every plugin, requirements and data
        cls.load_setup(setup, level=0)

        
            # Run the mode
48
        if getattr(getattr(setup, 'mode', None), 'loaded_requirements', False):
49
            return setup.mode.execute(**args)
50
        
51
52
53
54
        else:
            verbose('pycif has correctly been initialized '
                    'but no execution mode was specified')
    
55
    
56
57
    @classmethod
    def from_yaml(cls, def_fic):
Antoine Berchet's avatar
Antoine Berchet committed
58
        """Generates a dictionary including all pyCIF parameters
59
60
61
62
63
64
        
        Args:
            def_fic (string) : Path to the definition file
                                Handles both absolute and relative paths
    
        Returns:
Antoine Berchet's avatar
Antoine Berchet committed
65
            config_dict (dictionary): Dictionary populated with all pyCIF
66
            parameters
67
68
69
    
        """
        
70
        fic = os.path.abspath(os.path.expanduser(def_fic))
71
72
73
        
        try:
            with open(fic, 'r') as f:
Espen Sollum's avatar
Espen Sollum committed
74
                config_dict = yaml.load(f, Loader=yaml.FullLoader)
75
76
77
78
79
80
81
82
                
                config_dict['def_file'] = fic
                
                # Converting dates to datetime if necessary
                config_dict['datei'] = dates.date2datetime(config_dict['datei'])
                config_dict['datef'] = dates.date2datetime(config_dict['datef'])
                
                return config_dict
83
        
84
85
86
87
        except IOError as e:
            verbose("Couldn't find config file: {}".format(fic))
            verbose("Please check directories")
            raise e
88
        
89
90
91
        except yaml.scanner.ScannerError as e:
            verbose("Error in the syntax of config file: {}".format(fic))
            raise e
Joel Thanwerdas's avatar
Joel Thanwerdas committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

    @classmethod
    def yaml_subconfigs(cls, config_dict):
        for key, value in config_dict.iteritems():
            if isinstance(value, dict):
                config_dict[key] = cls.yaml_subconfigs(value)
            else:
                if key == 'fic_yaml':
                    if not os.path.isfile(value):
                        raise OSError('The Yaml path given is not a file : '
                                      '{}'.format(value))
                    if not os.path.exists(value):
                        raise OSError('The Yaml path given is not valid '
                                      '{}'.format(value))
                    with open(value, 'r') as f:
                        subconfig_dict = yaml.load(f)
                        config_dict = subconfig_dict
        return config_dict

111
112
    @classmethod
    def config_verbose(cls, config_dict):
Antoine Berchet's avatar
Antoine Berchet committed
113
        """Prints out main input parameters for pyCIF
114
        """
115
        
116
        verbose_txt = [
Antoine Berchet's avatar
Antoine Berchet committed
117
            "pyCIF has been initialized with the following parameters:",
118
119
120
121
122
123
            "Yaml configuration file: {}".format(config_dict['def_file']),
            "Log file: {}".format(config_dict['logfile']),
            "Start date: {}".format(config_dict['datei']),
            "End date: {}".format(config_dict['datef']),
            "Working directory: {}".format(config_dict['workdir']),
        ]
124
        
125
126
        map(lambda v: check.verbose(v), verbose_txt)
    
127
128
129
    @classmethod
    def load_setup(cls, plg,
                   parent_plg_type=None, level=None, **kwargs):
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        """Loads a Setup plugin.
        Loops recursively over all attributes of the setup to load:
        1) sub-plugins are initialized as Plugin child-class templates (
        Domain, ObsVect, Model, etc);
        2) instances are saved to the Plugin class to be accessible for
        anywhere later one.
        
        This allows modifications of the data of a given plugin at some place
        of the code to be automatically forwarded to the rest of the code
        
        Args:
            self (Setup): the setup to load
            parent_plg_type (str): the last recognized plugin type that is
            inherited by children
        
        """
        
147
148
149
150
        # Update orig_dict if not yet defined
        if level == 0:
            # Saves level 0 entries as reference plugins in requirements
            cls._save_refplugins(plg)
151
        
152
        # Loop over self attributes and load them as other Class if necessary
153
154
155
156
157
        # If an argument 'todo_init' was specified, initialize only listed plg
        if 'todo_init' in cls._get_attribute_list(plg):
            attributes = plg.todo_init
        
        else:
158
159
160
161
162
163
            attributes = [a for a in cls._get_attribute_list(plg)
                          if a != 'plugin']

        # Keep in memory the root plg_type
        root_plg_type = parent_plg_type

164
165
        for attr in attributes:
            plg_attr = getattr(plg, attr)
166
167

            # Re-initializing parent type to the root
168
            parent_plg_type = root_plg_type
169

170
171
172
173
174
175
176
177
            # For reference instances, check whether the Plugin was already
            # initialized as requirement; if so, just take it from reference
            if attr in cls.reference_instances \
                    and getattr(plg_attr, 'isreference', False) \
                    and getattr(cls.reference_instances.get(attr, None),
                                'loaded_class', False):
                setattr(plg, attr, cls.reference_instances[attr])
                continue
178
            
179
180
181
182
183
184
185
186
            # If not a Plugin, continue
            if not issubclass(type(plg_attr), Plugin):
                continue
            
            # If is still a Setup class, means that should be processed and
            # Initialized
            if isinstance(plg_attr, Setup) \
                    and not getattr(plg_attr, 'loaded_class', False):
187
188
189
190
                # Load the plugin type depending on the attribute name
                # Do nothing if the attribute is named 'plugin'
                if attr != 'plugin':
                    parent_plg_type = \
191
192
                        plg_attr._load_plugin_type(attr,
                                                   parent_plg_type)
193

194
195
196
197
                # Build a child sub-class and
                # overwrite the Setup class if needed
                plg_attr = cls.childclass_factory(plg_attr,
                                                  child_type=parent_plg_type)
198
                
199
200
                # Keep in memory that the current attribute class is loaded
                plg_attr.loaded_class = True
201
            
202
203
204
205
206
207
208
209
210
211
            # Load all attributes recursively if not already done
            if not getattr(plg_attr, 'loaded_attributes', False):
                cls.load_setup(plg_attr, parent_plg_type,
                               **kwargs)
                plg_attr.loaded_attributes = True
            
            # Initializes the plugin from registered module if any
            if hasattr(plg_attr, 'initiate_template') \
                    and not getattr(plg_attr, 'loaded_template', False):
                plg_attr.initiate_template()
212

213
214
215
                # Saves the plugin to the class,
                # so it is accessible by everyone anywhere
                # (including its attributes and stored data)
216
217
218
219
220
221
222
223
                if hasattr(plg_attr, 'plugin'):
                    name = plg_attr.plugin.name
                    version = plg_attr.plugin.version
                    plg_type = plg_attr.plugin.type
                    
                    if not cls.is_loaded(name, version, plg_type) \
                            and name is not None:
                        cls.save_loaded(plg_attr)
224
                
225
                plg_attr.loaded_template = True
226
            
227
228
229
230
231
232
233
            # If requirements are not already loaded
            if not getattr(plg_attr, 'loaded_requirements', False):
                # Load requirements
                cls._check_requirements(plg_attr, **kwargs)
                
                # The plugin has been correctly loaded at this point
                plg_attr.loaded_requirements = True
234
            
235
236
237
238
239
            # Initializes the plugin data
            if hasattr(plg_attr, 'ini_data') \
                    and not getattr(plg_attr, 'loaded_data', False):
                plg_attr.ini_data(**kwargs)
                plg_attr.loaded_data = True
240
            
241
242
243
            # Linking present plugin to reference level 0 if needed
            if getattr(plg_attr, 'isreference', False):
                plg.reference_instances[attr] = plg_attr
244
            
245
246
            # Attach plugin to the parent plugin
            setattr(plg, attr, plg_attr)
247
    
248
    @classmethod
249
    def _check_requirements(cls, plg, **kwargs):
250
251
252
253
254
        """Checking that required modules and plugins are loaded.
        If not, load them.

        Requirements are defined in the __init__.py file of the
        corresponding plugin module.
255

256
257
        Args:
            plg (Plugin): a plugin to initialize
258

259
260
261
262
263
        Notes: Some basic parameters are added as requirements to all plugins;
        These are:
            'datei', 'datef', 'workdir', 'logfile', 'verbose'

        """
264
        
265
266
267
268
269
        # Dealing with default requirements supposed to be given at level 0
        for key in plg.default_requirements:
            if key not in cls._get_attribute_list(plg):
                if key in cls.reference_instances:
                    setattr(plg, key, cls.reference_instances[key])
270
                
271
272
273
274
275
                else:
                    raise PluginError("The default key '{}' is not prescribed"
                                      "neither in the plugin {}, nor in the "
                                      "level 0 of the configuration file"
                                      .format(key, plg))
276
        
277
278
        # Looping over requirements and including them
        for key in plg.requirements:
279
            key_req = plg.requirements[key]
280
281
            fromany = key_req.get('any', False)
            empty = key_req.get('empty')
282
            
283
284
285
            name = key_req.get('name', None)
            version = key_req.get('version', '')
            plg_type = key_req.get('type', key)
286
            newplg = key_req.get('newplg', False)
287
            
288
            # If not from any plugin, but no default value specified, error
289
290
291
292
293
294
            if not fromany and name is None:
                raise PluginError(
                    "{} needs a specific {}, but none was specified \n"
                    "Please check requirements in your module"
                        .format(plg, key)
                )
295
            
296
297
298
            # If needs a Plugin explicitly defined,
            # look for it at level 0 of setup, or in children
            plg_out = None
299
            
300
301
302
303
304
305
            if fromany:
                # If in children
                if key in cls._get_attribute_list(plg):
                    plg_tmp = getattr(plg, key)
                    
                    # If child has a prescribed name
306
                    if getattr(getattr(plg_tmp, 'plugin', None), 'name', None) \
307
                            is not None:
308
309
                        plg_out = plg_tmp
                    
310
311
312
313
314
315
316
317
                    # If a default is defined, load from registered
                    elif name is not None:
                        plg_out = cls.load_registered(name, version, plg_type,
                                                      plg_orig=plg_tmp)
                    
                    # Otherwise, if accepts empty classes
                    elif empty:
                        plg_out = cls.get_subclass(plg_type)(plg_orig=plg_tmp)
318
                    
319
                    # Error in the yaml if reaching this point
320
                    else:
321
                        raise PluginError(
322
323
                            "{} needs a plugin '{}/{}/{}' and an "
                            "inconsistent one was proposed in the Yaml"
324
                                .format(plg, key, name, version)
325
                        )
326
327
328
329
330
331
                
                # If not in children but at level 0 of Yaml
                elif key in cls.reference_instances:
                    plg_tmp = cls.reference_instances[key]
                    
                    # If reference has a prescribed name
332
                    if getattr(getattr(plg_tmp, 'plugin', None), 'name', None) \
333
334
335
336
337
338
339
                            is not None:
                        plg_out = plg_tmp
                    
                    # If a default is defined, load from registered
                    elif name is not None:
                        plg_out = cls.load_registered(name, version, plg_type,
                                                      plg_orig=plg_tmp)
340
                    
341
342
343
                    # Otherwise, if accepts empty classes
                    elif empty:
                        plg_out = cls.get_subclass(plg_type)(plg_orig=plg_tmp)
344
                    
345
346
347
                    # Error in the yaml if reaching this point
                    else:
                        raise PluginError(
348
349
                            "{} needs a plugin '{}/{}/{}' and an "
                            "inconsistent one was proposed in the Yaml"
350
                                .format(plg, key, name, version)
351
                        )
352
                
353
                elif empty:
354
355
356
357
358
                    if cls.is_registered(name, version, plg_type):
                        plg_out = cls.load_registered(name, version, plg_type)
                    
                    else:
                        plg_out = cls.get_subclass(plg_type)()
359
            
360
361
362
363
364
            # If needs a specific one
            else:
                # If in children
                if key in cls._get_attribute_list(plg):
                    plg_tmp = getattr(plg, key)
365
                    
366
367
368
369
370
                    # If child already of correct type, and correct ID
                    if plg_tmp.plugin.name is not None \
                            and name == plg_tmp.plugin.name \
                            and version == plg_tmp.plugin.version \
                            and type(plg_tmp) == cls.get_subclass(plg_type):
371
                        plg_out = plg_tmp
372
                    
373
374
375
376
377
                    # If a default is defined, load from registered
                    elif plg_tmp.plugin.name is None:
                        plg_out = cls.load_registered(name, version, plg_type,
                                                      plg_orig=plg_tmp)
                    
378
                    else:
379
380
381
382
383
                        raise PluginError(
                            "{} needs a specific plugin '{}/{}/{}' and a "
                            "different was specified in children"
                                .format(plg, key, name, version)
                        )
384
                
385
386
387
388
389
390
391
                # If not in children but at level 0 of Yaml
                elif key in cls.reference_instances:
                    plg_tmp = cls.reference_instances[key]
                    
                    # If child already of correct type, and correct ID
                    if plg_tmp.plugin.name is not None \
                            and name == plg_tmp.plugin.name \
392
393
                            and version == plg_tmp.plugin.version \
                            and not newplg:
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
                        plg_out = plg_tmp
                    
                    # If a default is defined, load from registered
                    elif plg_tmp.plugin.name is None:
                        cls.reference_instances[key] = \
                            cls.load_registered(name, version, plg_type,
                                                plg_orig=plg_tmp)
                        plg_out = cls.reference_instances[key]
                    
                    else:
                        
                        raise PluginError(
                            "{} needs a specific plugin '{}/{}/{}' and a "
                            "different was specified at level 0"
                                .format(plg, key, name, version)
                        )
                
                # If accepts empty class
412
                elif empty:
413
                    plg_out = cls.load_registered(name, version, plg_type)
414
            
415
            if plg_out is None:
416
                raise Exception(
417
418
419
420
421
                    "{} needs a Plugin '{}' to run properly\n"
                    "there is none in its children nor at the level 0 of "
                    "Yaml\n"
                    "Please check your Yaml"
                        .format(plg, key)
422
                )
423
424
425
426
427

            # Keep in memory to initialize a new instance of the plugin or not
            plg.plugin.newplg = newplg

            # Attaching the requirement to the parent plugin
428
            setattr(plg, key, plg_out)
429

430
        # Load the requirements if not already done
431
        cls.load_setup(plg, **kwargs)