Skip to content

Saving Care model twice #62

Description

@jdeschamps

Hi!

I encounter issues when trying to save twice the models from the notebooks. The first run of the cell exports the model without problem.

In CSBDeep, denoising2D notebook:

model.export_TF()
model.export_TF()

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
Cell In [13], line 1
----> 1 model.export_TF()

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/models/base_model.py:32, in suppress_without_basedir.<locals>._suppress_without_basedir.<locals>.wrapper(*args, **kwargs)
     30     warn is False or warnings.warn("Suppressing call of '%s' (due to basedir=None)." % f.__name__)
     31 else:
---> 32     return f(*args, **kwargs)

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/models/care_standard.py:228, in CARE.export_TF(self, fname)
    218     fname = Path(fname)
    220 meta = {
    221     'type':          self.__class__.__name__,
    222     'version':       package_version,
   (...)
    226     'tile_overlap':  self._axes_tile_overlap(self.config.axes),
    227 }
--> 228 export_SavedModel(self.keras_model, str(fname), meta=meta)
    229 print("\nModel exported in TensorFlow's SavedModel format:\n%s" % str(fname.resolve()))

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/utils/tf.py:230, in export_SavedModel(model, outpath, meta, format)
    228 with tempfile.TemporaryDirectory() as tmpdir:
    229     tmpsubdir = os.path.join(tmpdir,'model')
--> 230     export_to_dir(tmpsubdir)
    231     shutil.make_archive(outpath, format, tmpsubdir)

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/utils/tf.py:204, in export_SavedModel.<locals>.export_to_dir(dirname)
    202 else:
    203     from tensorflow.keras.models import clone_model
--> 204     weights = model.get_weights()
    205     with tf.Graph().as_default():
    206         # clone model in new graph and set weights
    207         _model = clone_model(model)

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/keras/engine/training_v1.py:160, in Model.get_weights(self)
    152 def get_weights(self):
    153     """Retrieves the weights of the model.
    154 
    155     Returns:
    156         A flat list of Numpy arrays.
    157     """
    158     strategy = (
    159         self._distribution_strategy
--> 160         or self._compile_time_distribution_strategy
    161     )
    162     if strategy:
    163         with strategy.scope():

AttributeError: 'Functional' object has no attribute '_compile_time_distribution_strategy'

Originally, I encountered a different problem in n2v (see juglab/n2v#130) where the custom export_TF method calls:

def export_SavedModel(model, outpath, meta={}, format='zip'):

and calling export_TF twice leads to the following error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In [15], line 1
----> 1 model.export_TF(name='Noise2Void - 2D SEM Example', 
      2                 description='This is the 2D Noise2Void example trained on SEM data in python.', 
      3                 authors=["Tim-Oliver Buchholz", "Alexander Krull", "Florian Jug"],
      4                 test_img=X_val[0,...,0], axes='YX',
      5                 patch_shape=patch_shape)

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/models/base_model.py:32, in suppress_without_basedir.<locals>._suppress_without_basedir.<locals>.wrapper(*args, **kwargs)
     30     warn is False or warnings.warn("Suppressing call of '%s' (due to basedir=None)." % f.__name__)
     31 else:
---> 32     return f(*args, **kwargs)

File ~/git/n2v/n2v/models/n2v_standard.py:460, in N2V.export_TF(self, name, description, authors, test_img, axes, patch_shape, fname)
    457 assert input_n_dims == self.config.n_dim, 'Input and network dimensions do not match.'
    458 assert test_img.shape[axes.index('X')] == test_img.shape[
    459     axes.index('Y')], 'X and Y dimensions are not of same length.'
--> 460 test_output = self.predict(test_img, axes)
    461 # Extract central slice of Z-Stack
    462 if 'Z' in axes:

File ~/git/n2v/n2v/models/n2v_standard.py:408, in N2V.predict(self, img, axes, resizer, n_tiles, tta)
    405     pred = tta_backward(preds)
    406 else:
    407     pred = \
--> 408         self._predict_mean_and_scale(normalized, axes=new_axes, normalizer=None, resizer=resizer,
    409                                      n_tiles=new_n_tiles)[0]
    411 pred = self.__denormalize__(pred, means, stds)
    413 if 'C' in axes:

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/models/care_standard.py:377, in CARE._predict_mean_and_scale(self, img, axes, normalizer, resizer, n_tiles)
    374 while not done:
    375     try:
    376         # raise tf.errors.ResourceExhaustedError(None,None,None) # tmp
--> 377         x = predict_tiled(self.keras_model,x,axes_in=net_axes_in,axes_out=net_axes_out,
    378                           n_tiles=n_tiles,block_sizes=net_axes_in_div_by,tile_overlaps=net_axes_in_overlaps,pbar=progress)
    379         # x has net_axes_out semantics
    380         done = True

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/internals/predict.py:51, in predict_tiled(keras_model, x, n_tiles, block_sizes, tile_overlaps, axes_in, axes_out, pbar, **kwargs)
     48 """TODO."""
     50 if all(t==1 for t in n_tiles):
---> 51     pred = predict_direct(keras_model,x,axes_in,axes_out,**kwargs)
     52     if pbar is not None:
     53         pbar.update()

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/internals/predict.py:41, in predict_direct(keras_model, x, axes_in, axes_out, **kwargs)
     39 len(axes_in) == x.ndim or _raise(ValueError())
     40 x = to_tensor(x,channel=channel_in,single_sample=single_sample)
---> 41 pred = from_tensor(keras_model.predict(x,**kwargs),channel=channel_out,single_sample=single_sample)
     42 len(axes_out) == pred.ndim or _raise(ValueError())
     43 return pred

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/keras/engine/training_v1.py:1053, in Model.predict(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)
    988 def predict(
    989     self,
    990     x,
   (...)
    997     use_multiprocessing=False,
    998 ):
    999     """Generates output predictions for the input samples.
   1000 
   1001     Computation is done in batches (see the `batch_size` arg.)
   (...)
   1051             that is not a multiple of the batch size.
   1052     """
-> 1053     self._assert_built_as_v1()
   1054     base_layer.keras_api_gauge.get_cell("predict").set(True)
   1055     self._check_call_args("predict")

File ~/miniconda3/envs/n2v2/lib/python3.9/site-packages/keras/engine/base_layer_v1.py:906, in Layer._assert_built_as_v1(self)
    904 def _assert_built_as_v1(self):
    905     if not hasattr(self, "_originally_built_as_v1"):
--> 906         raise ValueError(
    907             "Your Layer or Model is in an invalid state. "
    908             "This can happen for the following cases:\n "
    909             "1. You might be interleaving estimator/non-estimator models "
    910             "or interleaving models/layers made in "
    911             "tf.compat.v1.Graph.as_default() with models/layers created "
    912             "outside of it. "
    913             "Converting a model to an estimator (via model_to_estimator) "
    914             "invalidates all models/layers made before the conversion "
    915             "(even if they were not the model converted to an estimator). "
    916             "Similarly, making a layer or a model inside a "
    917             "a tf.compat.v1.Graph invalidates all layers/models you "
    918             "previously made outside of the graph.\n"
    919             "2. You might be using a custom keras layer implementation "
    920             "with custom __init__ which didn't call super().__init__. "
    921             " Please check the implementation of %s and its bases."
    922             % (type(self),)
    923         )

ValueError: Your Layer or Model is in an invalid state. This can happen for the following cases:
 1. You might be interleaving estimator/non-estimator models or interleaving models/layers made in 
tf.compat.v1.Graph.as_default() with models/layers created outside of it. Converting a model to an estimator (via model_to_estimator) invalidates all models/layers made before the conversion (even if they were not the model converted to an estimator). Similarly, making a layer or a model inside a a tf.compat.v1.Graph invalidates all layers/models you previously made outside of the graph.
3. You might be using a custom keras layer implementation with custom __init__ which didn't call super().__init__.  Please check the implementation of <class 'keras.engine.functional.Functional'> and its bases.

If I call CSBdeep export_SavedModel instead within one of the N2V notebook, I get the same error:

from csbdeep.utils.tf import export_SavedModel
fname = model.logdir / 'export.bioimage.io.zip'
meta = {
    'type': model.__class__.__name__,
    'version': '1.0',
    'probabilistic': model.config.probabilistic,
    'axes': model.config.axes,
    'axes_div_by': model._axes_div_by(model.config.axes),
    'tile_overlap': model._axes_tile_overlap(model.config.axes),
}
export_SavedModel(model.keras_model, str(fname), meta=meta)
export_SavedModel(model.keras_model, str(fname), meta=meta)


  warnings.warn(\
WARNING:tensorflow:From /home/joran.deschamps/miniconda3/envs/n2v2/lib/python3.9/site-packages/tensorflow/python/saved_model/signature_def_utils_impl.py:203: build_tensor_info (from tensorflow.python.saved_model.utils_impl) is deprecated and will be removed in a future version.
Instructions for updating:
This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.utils.build_tensor_info or tf.compat.v1.saved_model.build_tensor_info.
Traceback (most recent call last):
  File "/home/joran.deschamps/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/utils/tf.py", line 230, in export_SavedModel
    export_to_dir(tmpsubdir)
  File "/home/joran.deschamps/miniconda3/envs/n2v2/lib/python3.9/site-packages/csbdeep/utils/tf.py", line 204, in export_to_dir
    weights = model.get_weights()
  File "/home/joran.deschamps/miniconda3/envs/n2v2/lib/python3.9/site-packages/keras/engine/training_v1.py", line 160, in get_weights
    or self._compile_time_distribution_strategy
AttributeError: 'Functional' object has no attribute '_compile_time_distribution_strategy'

The env I am using has the following:

tensorflow                2.10.0                   pypi_0    pypi
tensorflow-estimator      2.10.0                   pypi_0    pypi
tensorflow-io-gcs-filesystem 0.27.0                   pypi_0    pypi
keras                     2.10.0                   pypi_0    pypi
keras-preprocessing       1.1.2                    pypi_0    pypi
csbdeep                   0.7.2                    pypi_0    pypi

Have you encountered this before?

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Fields

    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions