# Copyright The PyTorch Lightning team.## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.# You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0## Unless required by applicable law or agreed to in writing, software# distributed under the License is distributed on an "AS IS" BASIS,# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.# See the License for the specific language governing permissions and# limitations under the License."""Neptune Logger--------------"""__all__=["NeptuneLogger",]importloggingimportosimportwarningsfromargparseimportNamespacefromfunctoolsimportreducefromtypingimportAny,Callable,Dict,Generator,Mapping,Optional,Sequence,Set,UnionfromweakrefimportReferenceTypeimporttorchfromtorchimportTensorfrompytorch_lightningimport__version__frompytorch_lightning.callbacksimportCheckpointfrompytorch_lightning.loggers.loggerimportLogger,rank_zero_experimentfrompytorch_lightning.utilities.importsimport_RequirementAvailablefrompytorch_lightning.utilities.loggerimport_add_prefix,_convert_params,_sanitize_callable_paramsfrompytorch_lightning.utilities.model_summaryimportModelSummaryfrompytorch_lightning.utilities.rank_zeroimportrank_zero_only_NEPTUNE_AVAILABLE=_RequirementAvailable("neptune-client")_NEPTUNE_GREATER_EQUAL_0_9=_RequirementAvailable("neptune-client>=0.9.0")if_NEPTUNE_AVAILABLEand_NEPTUNE_GREATER_EQUAL_0_9:try:fromneptuneimportnewasneptunefromneptune.new.exceptionsimportNeptuneLegacyProjectException,NeptuneOfflineModeFetchExceptionfromneptune.new.runimportRunfromneptune.new.typesimportFileasNeptuneFileexceptModuleNotFoundError:importneptunefromneptune.exceptionsimportNeptuneLegacyProjectException,NeptuneOfflineModeFetchExceptionfromneptune.runimportRunfromneptune.typesimportFileasNeptuneFileelse:# needed for test mocks, and function signaturesneptune,Run,NeptuneFile=None,None,Nonelog=logging.getLogger(__name__)_INTEGRATION_VERSION_KEY="source_code/integrations/pytorch-lightning"# kwargs used in previous NeptuneLogger version, now deprecated_LEGACY_NEPTUNE_INIT_KWARGS=["project_name","offline_mode","experiment_name","experiment_id","params","properties","upload_source_files","abort_callback","logger","upload_stdout","upload_stderr","send_hardware_metrics","run_monitoring_thread","handle_uncaught_exceptions","git_info","hostname","notebook_id","notebook_path",]# kwargs used in legacy NeptuneLogger from neptune-pytorch-lightning package_LEGACY_NEPTUNE_LOGGER_KWARGS=["base_namespace","close_after_fit",]
[docs]classNeptuneLogger(Logger):r""" Log using `Neptune <https://neptune.ai>`_. Install it with pip: .. code-block:: bash pip install neptune-client or conda: .. code-block:: bash conda install -c conda-forge neptune-client **Quickstart** Pass NeptuneLogger instance to the Trainer to log metadata with Neptune: .. code-block:: python from pytorch_lightning import Trainer from pytorch_lightning.loggers import NeptuneLogger neptune_logger = NeptuneLogger( api_key="ANONYMOUS", # replace with your own project="common/pytorch-lightning-integration", # format "<WORKSPACE/PROJECT>" tags=["training", "resnet"], # optional ) trainer = Trainer(max_epochs=10, logger=neptune_logger) **How to use NeptuneLogger?** Use the logger anywhere in your :class:`~pytorch_lightning.core.module.LightningModule` as follows: .. code-block:: python from neptune.new.types import File from pytorch_lightning import LightningModule class LitModel(LightningModule): def training_step(self, batch, batch_idx): # log metrics acc = ... self.log("train/loss", loss) def any_lightning_module_function_or_hook(self): # log images img = ... self.logger.experiment["train/misclassified_images"].log(File.as_image(img)) # generic recipe metadata = ... self.logger.experiment["your/metadata/structure"].log(metadata) Note that syntax: ``self.logger.experiment["your/metadata/structure"].log(metadata)`` is specific to Neptune and it extends logger capabilities. Specifically, it allows you to log various types of metadata like scores, files, images, interactive visuals, CSVs, etc. Refer to the `Neptune docs <https://docs.neptune.ai/you-should-know/logging-metadata#essential-logging-methods>`_ for more detailed explanations. You can also use regular logger methods ``log_metrics()``, and ``log_hyperparams()`` with NeptuneLogger as these are also supported. **Log after fitting or testing is finished** You can log objects after the fitting or testing methods are finished: .. code-block:: python neptune_logger = NeptuneLogger(project="common/pytorch-lightning-integration") trainer = pl.Trainer(logger=neptune_logger) model = ... datamodule = ... trainer.fit(model, datamodule=datamodule) trainer.test(model, datamodule=datamodule) # Log objects after `fit` or `test` methods # model summary neptune_logger.log_model_summary(model=model, max_depth=-1) # generic recipe metadata = ... neptune_logger.experiment["your/metadata/structure"].log(metadata) **Log model checkpoints** If you have :class:`~pytorch_lightning.callbacks.ModelCheckpoint` configured, Neptune logger automatically logs model checkpoints. Model weights will be uploaded to the: "model/checkpoints" namespace in the Neptune Run. You can disable this option: .. code-block:: python neptune_logger = NeptuneLogger(project="common/pytorch-lightning-integration", log_model_checkpoints=False) **Pass additional parameters to the Neptune run** You can also pass ``neptune_run_kwargs`` to specify the run in the greater detail, like ``tags`` or ``description``: .. testcode:: from pytorch_lightning import Trainer from pytorch_lightning.loggers import NeptuneLogger neptune_logger = NeptuneLogger( project="common/pytorch-lightning-integration", name="lightning-run", description="mlp quick run with pytorch-lightning", tags=["mlp", "quick-run"], ) trainer = Trainer(max_epochs=3, logger=neptune_logger) Check `run documentation <https://docs.neptune.ai/essentials/api-reference/run>`_ for more info about additional run parameters. **Details about Neptune run structure** Runs can be viewed as nested dictionary-like structures that you can define in your code. Thanks to this you can easily organize your metadata in a way that is most convenient for you. The hierarchical structure that you apply to your metadata will be reflected later in the UI. You can organize this way any type of metadata - images, parameters, metrics, model checkpoint, CSV files, etc. See Also: - Read about `what object you can log to Neptune <https://docs.neptune.ai/you-should-know/what-can-you-log-and-display>`_. - Check `example run <https://app.neptune.ai/o/common/org/pytorch-lightning-integration/e/PTL-1/all>`_ with multiple types of metadata logged. - For more detailed info check `user guide <https://docs.neptune.ai/integrations-and-supported-tools/model-training/pytorch-lightning>`_. Args: api_key: Optional. Neptune API token, found on https://neptune.ai upon registration. Read: `how to find and set Neptune API token <https://docs.neptune.ai/administration/security-and-privacy/ how-to-find-and-set-neptune-api-token>`_. It is recommended to keep it in the `NEPTUNE_API_TOKEN` environment variable and then you can drop ``api_key=None``. project: Optional. Name of a project in a form of "my_workspace/my_project" for example "tom/mask-rcnn". If ``None``, the value of `NEPTUNE_PROJECT` environment variable will be taken. You need to create the project in https://neptune.ai first. name: Optional. Editable name of the run. Run name appears in the "all metadata/sys" section in Neptune UI. run: Optional. Default is ``None``. The Neptune ``Run`` object. If specified, this `Run`` will be used for logging, instead of a new Run. When run object is passed you can't specify other neptune properties. log_model_checkpoints: Optional. Default is ``True``. Log model checkpoint to Neptune. Works only if ``ModelCheckpoint`` is passed to the ``Trainer``. prefix: Optional. Default is ``"training"``. Root namespace for all metadata logging. \**neptune_run_kwargs: Additional arguments like ``tags``, ``description``, ``capture_stdout``, etc. used when run is created. Raises: ModuleNotFoundError: If required Neptune package in version >=0.9 is not installed on the device. TypeError: If configured project has not been migrated to new structure yet. ValueError: If argument passed to the logger's constructor is incorrect. """LOGGER_JOIN_CHAR="/"PARAMETERS_KEY="hyperparams"ARTIFACTS_KEY="artifacts"def__init__(self,*,# force users to call `NeptuneLogger` initializer with `kwargs`api_key:Optional[str]=None,project:Optional[str]=None,name:Optional[str]=None,run:Optional["Run"]=None,log_model_checkpoints:Optional[bool]=True,prefix:str="training",agg_key_funcs:Optional[Mapping[str,Callable[[Sequence[float]],float]]]=None,agg_default_func:Optional[Callable[[Sequence[float]],float]]=None,**neptune_run_kwargs,):# verify if user passed proper init argumentsself._verify_input_arguments(api_key,project,name,run,neptune_run_kwargs)ifneptuneisNone:raiseModuleNotFoundError("You want to use the `Neptune` logger which is not installed yet, install it with"" `pip install neptune-client`.")super().__init__(agg_key_funcs=agg_key_funcs,agg_default_func=agg_default_func)self._log_model_checkpoints=log_model_checkpointsself._prefix=prefixself._run_name=nameself._project_name=projectself._api_key=api_keyself._run_instance=runself._neptune_run_kwargs=neptune_run_kwargsself._run_short_id=Noneifself._run_instanceisnotNone:self._retrieve_run_data()# make sure that we've log integration version for outside `Run` instancesself._run_instance[_INTEGRATION_VERSION_KEY]=__version__def_retrieve_run_data(self):try:self._run_instance.wait()self._run_short_id=self._run_instance["sys/id"].fetch()self._run_name=self._run_instance["sys/name"].fetch()exceptNeptuneOfflineModeFetchException:self._run_short_id="OFFLINE"self._run_name="offline-name"@propertydef_neptune_init_args(self):args={}# Backward compatibility in case of previous version retrievaltry:args=self._neptune_run_kwargsexceptAttributeError:passifself._project_nameisnotNone:args["project"]=self._project_nameifself._api_keyisnotNone:args["api_token"]=self._api_keyifself._run_short_idisnotNone:args["run"]=self._run_short_id# Backward compatibility in case of previous version retrievaltry:ifself._run_nameisnotNone:args["name"]=self._run_nameexceptAttributeError:passreturnargsdef_construct_path_with_prefix(self,*keys)->str:"""Return sequence of keys joined by `LOGGER_JOIN_CHAR`, started with `_prefix` if defined."""ifself._prefix:returnself.LOGGER_JOIN_CHAR.join([self._prefix,*keys])returnself.LOGGER_JOIN_CHAR.join(keys)@staticmethoddef_verify_input_arguments(api_key:Optional[str],project:Optional[str],name:Optional[str],run:Optional["Run"],neptune_run_kwargs:dict,):legacy_kwargs_msg=("Following kwargs are deprecated: {legacy_kwargs}.\n""If you are looking for the Neptune logger using legacy Python API,"" it's still available as part of neptune-contrib package:\n"" - https://docs-legacy.neptune.ai/integrations/pytorch_lightning.html\n""The NeptuneLogger was re-written to use the neptune.new Python API\n"" - https://neptune.ai/blog/neptune-new\n"" - https://docs.neptune.ai/integrations-and-supported-tools/model-training/pytorch-lightning\n""You should use arguments accepted by either NeptuneLogger.init() or neptune.init()")# check if user used legacy kwargs expected in `NeptuneLegacyLogger`used_legacy_kwargs=[legacy_kwargforlegacy_kwarginneptune_run_kwargsiflegacy_kwargin_LEGACY_NEPTUNE_INIT_KWARGS]ifused_legacy_kwargs:raiseValueError(legacy_kwargs_msg.format(legacy_kwargs=used_legacy_kwargs))# check if user used legacy kwargs expected in `NeptuneLogger` from neptune-pytorch-lightning packageused_legacy_neptune_kwargs=[legacy_kwargforlegacy_kwarginneptune_run_kwargsiflegacy_kwargin_LEGACY_NEPTUNE_LOGGER_KWARGS]ifused_legacy_neptune_kwargs:raiseValueError(legacy_kwargs_msg.format(legacy_kwargs=used_legacy_neptune_kwargs))# check if user passed new client `Run` objectifrunisnotNoneandnotisinstance(run,Run):raiseValueError("Run parameter expected to be of type `neptune.new.Run`.\n""If you are looking for the Neptune logger using legacy Python API,"" it's still available as part of neptune-contrib package:\n"" - https://docs-legacy.neptune.ai/integrations/pytorch_lightning.html\n""The NeptuneLogger was re-written to use the neptune.new Python API\n"" - https://neptune.ai/blog/neptune-new\n"" - https://docs.neptune.ai/integrations-and-supported-tools/model-training/pytorch-lightning\n")# check if user passed redundant neptune.init arguments when passed runany_neptune_init_arg_passed=any(argisnotNoneforargin[api_key,project,name])orneptune_run_kwargsifrunisnotNoneandany_neptune_init_arg_passed:raiseValueError("When an already initialized run object is provided"" you can't provide other neptune.init() parameters.\n")def__getstate__(self):state=self.__dict__.copy()# Run instance can't be pickledstate["_run_instance"]=Nonereturnstatedef__setstate__(self,state):self.__dict__=stateself._run_instance=neptune.init(**self._neptune_init_args)@property@rank_zero_experimentdefexperiment(self)->Run:r""" Actual Neptune run object. Allows you to use neptune logging features in your :class:`~pytorch_lightning.core.module.LightningModule`. Example:: class LitModel(LightningModule): def training_step(self, batch, batch_idx): # log metrics acc = ... self.logger.experiment["train/acc"].log(acc) # log images img = ... self.logger.experiment["train/misclassified_images"].log(File.as_image(img)) Note that syntax: ``self.logger.experiment["your/metadata/structure"].log(metadata)`` is specific to Neptune and it extends logger capabilities. Specifically, it allows you to log various types of metadata like scores, files, images, interactive visuals, CSVs, etc. Refer to the `Neptune docs <https://docs.neptune.ai/you-should-know/logging-metadata#essential-logging-methods>`_ for more detailed explanations. You can also use regular logger methods ``log_metrics()``, and ``log_hyperparams()`` with NeptuneLogger as these are also supported. """returnself.run@property@rank_zero_experimentdefrun(self)->Run:try:ifnotself._run_instance:self._run_instance=neptune.init(**self._neptune_init_args)self._retrieve_run_data()# make sure that we've log integration version for newly createdself._run_instance[_INTEGRATION_VERSION_KEY]=__version__returnself._run_instanceexceptNeptuneLegacyProjectExceptionase:raiseTypeError(f"Project {self._project_name} has not been migrated to the new structure."" You can still integrate it with the Neptune logger using legacy Python API"" available as part of neptune-contrib package:"" https://docs-legacy.neptune.ai/integrations/pytorch_lightning.html\n")frome
[docs]@rank_zero_onlydeflog_hyperparams(self,params:Union[Dict[str,Any],Namespace])->None:# skipcq: PYL-W0221r""" Log hyper-parameters to the run. Hyperparams will be logged under the "<prefix>/hyperparams" namespace. Note: You can also log parameters by directly using the logger instance: ``neptune_logger.experiment["model/hyper-parameters"] = params_dict``. In this way you can keep hierarchical structure of the parameters. Args: params: `dict`. Python dictionary structure with parameters. Example:: from pytorch_lightning.loggers import NeptuneLogger PARAMS = { "batch_size": 64, "lr": 0.07, "decay_factor": 0.97 } neptune_logger = NeptuneLogger( api_key="ANONYMOUS", project="common/pytorch-lightning-integration" ) neptune_logger.log_hyperparams(PARAMS) """params=_convert_params(params)params=_sanitize_callable_params(params)parameters_key=self.PARAMETERS_KEYparameters_key=self._construct_path_with_prefix(parameters_key)self.run[parameters_key]=params
[docs]@rank_zero_onlydeflog_metrics(self,metrics:Dict[str,Union[Tensor,float]],step:Optional[int]=None)->None:"""Log metrics (numeric values) in Neptune runs. Args: metrics: Dictionary with metric names as keys and measured quantities as values. step: Step number at which the metrics should be recorded, currently ignored. """ifrank_zero_only.rank!=0:raiseValueError("run tried to log from global_rank != 0")metrics=_add_prefix(metrics,self._prefix,self.LOGGER_JOIN_CHAR)forkey,valinmetrics.items():# `step` is ignored because Neptune expects strictly increasing step values which# Lightning does not always guarantee.self.run[key].log(val)
@propertydefsave_dir(self)->Optional[str]:"""Gets the save directory of the experiment which in this case is ``None`` because Neptune does not save locally. Returns: the root directory where experiment logs get saved """returnos.path.join(os.getcwd(),".neptune")@rank_zero_onlydeflog_model_summary(self,model,max_depth=-1):model_str=str(ModelSummary(model=model,max_depth=max_depth))self.run[self._construct_path_with_prefix("model/summary")]=neptune.types.File.from_content(content=model_str,extension="txt")
[docs]@rank_zero_onlydefafter_save_checkpoint(self,checkpoint_callback:"ReferenceType[Checkpoint]")->None:"""Automatically log checkpointed model. Called after model checkpoint callback saves a new checkpoint. Args: checkpoint_callback: the model checkpoint callback instance """ifnotself._log_model_checkpoints:returnfile_names=set()checkpoints_namespace=self._construct_path_with_prefix("model/checkpoints")# save last modelifhasattr(checkpoint_callback,"last_model_path")andcheckpoint_callback.last_model_path:model_last_name=self._get_full_model_name(checkpoint_callback.last_model_path,checkpoint_callback)file_names.add(model_last_name)self.run[f"{checkpoints_namespace}/{model_last_name}"].upload(checkpoint_callback.last_model_path)# save best k modelsifhasattr(checkpoint_callback,"best_k_models"):forkeyincheckpoint_callback.best_k_models.keys():model_name=self._get_full_model_name(key,checkpoint_callback)file_names.add(model_name)self.run[f"{checkpoints_namespace}/{model_name}"].upload(key)# log best model path and checkpointifhasattr(checkpoint_callback,"best_model_path")andcheckpoint_callback.best_model_path:self.run[self._construct_path_with_prefix("model/best_model_path")]=checkpoint_callback.best_model_pathmodel_name=self._get_full_model_name(checkpoint_callback.best_model_path,checkpoint_callback)file_names.add(model_name)self.run[f"{checkpoints_namespace}/{model_name}"].upload(checkpoint_callback.best_model_path)# remove old models logged to experiment if they are not part of best k models at this pointifself.run.exists(checkpoints_namespace):exp_structure=self.run.get_structure()uploaded_model_names=self._get_full_model_names_from_exp_structure(exp_structure,checkpoints_namespace)forfile_to_dropinlist(uploaded_model_names-file_names):delself.run[f"{checkpoints_namespace}/{file_to_drop}"]# log best model scoreifhasattr(checkpoint_callback,"best_model_score")andcheckpoint_callback.best_model_score:self.run[self._construct_path_with_prefix("model/best_model_score")]=(checkpoint_callback.best_model_score.cpu().detach().numpy())
@staticmethoddef_get_full_model_name(model_path:str,checkpoint_callback:"ReferenceType[Checkpoint]")->str:"""Returns model name which is string `model_path` appended to `checkpoint_callback.dirpath`."""ifhasattr(checkpoint_callback,"dirpath"):expected_model_path=f"{checkpoint_callback.dirpath}{os.path.sep}"ifnotmodel_path.startswith(expected_model_path):raiseValueError(f"{model_path} was expected to start with {expected_model_path}.")# Remove extension from filepathfilepath,_=os.path.splitext(model_path[len(expected_model_path):])else:filepath=model_pathreturnfilepath@classmethoddef_get_full_model_names_from_exp_structure(cls,exp_structure:dict,namespace:str)->Set[str]:"""Returns all paths to properties which were already logged in `namespace`"""structure_keys=namespace.split(cls.LOGGER_JOIN_CHAR)uploaded_models_dict=reduce(lambdad,k:d[k],[exp_structure,*structure_keys])returnset(cls._dict_paths(uploaded_models_dict))@classmethoddef_dict_paths(cls,d:dict,path_in_build:str=None)->Generator:fork,vind.items():path=f"{path_in_build}/{k}"ifpath_in_buildisnotNoneelsekifnotisinstance(v,dict):yieldpathelse:yield fromcls._dict_paths(v,path)@propertydefname(self)->str:"""Return the experiment name or 'offline-name' when exp is run in offline mode."""returnself._run_name@propertydefversion(self)->str:"""Return the experiment version. It's Neptune Run's short_id """returnself._run_short_id@staticmethoddef_signal_deprecated_api_usage(f_name,sample_code,raise_exception=False):msg_suffix=(f"If you are looking for the Neptune logger using legacy Python API,"f" it's still available as part of neptune-contrib package:\n"f" - https://docs-legacy.neptune.ai/integrations/pytorch_lightning.html\n"f"The NeptuneLogger was re-written to use the neptune.new Python API\n"f" - https://neptune.ai/blog/neptune-new\n"f" - https://docs.neptune.ai/integrations-and-supported-tools/model-training/pytorch-lightning\n"f"Instead of `logger.{f_name}` you can use:\n"f"\t{sample_code}")ifnotraise_exception:warnings.warn("The function you've used is deprecated in v1.5.0 and will be removed in v1.7.0. "+msg_suffix)else:raiseValueError("The function you've used is deprecated.\n"+msg_suffix)@rank_zero_onlydeflog_metric(self,metric_name:str,metric_value:Union[Tensor,float,str],step:Optional[int]=None):key=f"{self._prefix}/{metric_name}"self._signal_deprecated_api_usage("log_metric",f"logger.run['{key}'].log(42)")iftorch.is_tensor(metric_value):metric_value=metric_value.cpu().detach()self.run[key].log(metric_value,step=step)@rank_zero_onlydeflog_text(self,log_name:str,text:str,step:Optional[int]=None)->None:key=f"{self._prefix}/{log_name}"self._signal_deprecated_api_usage("log_text",f"logger.run['{key}].log('text')")self.run[key].log(str(text),step=step)@rank_zero_onlydeflog_image(self,log_name:str,image:Union[str,Any],step:Optional[int]=None)->None:key=f"{self._prefix}/{log_name}"self._signal_deprecated_api_usage("log_image",f"logger.run['{key}'].log(File('path_to_image'))")ifisinstance(image,str):# if `img` is path to file, convert it to file objectimage=NeptuneFile(image)self.run[key].log(image,step=step)@rank_zero_onlydeflog_artifact(self,artifact:str,destination:Optional[str]=None)->None:key=f"{self._prefix}/{self.ARTIFACTS_KEY}/{artifact}"self._signal_deprecated_api_usage("log_artifact",f"logger.run['{key}].log('path_to_file')")self.run[key].log(destination)defset_property(self,*args,**kwargs):self._signal_deprecated_api_usage("log_artifact",f"logger.run['{self._prefix}/{self.PARAMETERS_KEY}/key'].log(value)",raise_exception=True)defappend_tags(self,*args,**kwargs):self._signal_deprecated_api_usage("append_tags","logger.run['sys/tags'].add(['foo', 'bar'])",raise_exception=True)
To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. Read PyTorch Lightning's Privacy Policy.