diff --git a/CHANGELOG.md b/CHANGELOG.md index 0534728d..3edf1472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## Version 1.3.1 - unreleased + +### Fixed + +- Fixed a bug where parameter groups were omitted when using `TransformerBasedClassification`'s layer-specific fine-tuning functionality. ([#36](https://github.com/webis-de/small-text/issues/36), [#38](https://github.com/webis-de/small-text/pull/38)) + +### Contributors + +[@JP-SystemsX](https://github.com/JP-SystemsX) + ## Version 1.3.0 - 2023-02-21 ### Added diff --git a/docs/libraries/transformers_integration.rst b/docs/libraries/transformers_integration.rst index d2996606..d3528743 100644 --- a/docs/libraries/transformers_integration.rst +++ b/docs/libraries/transformers_integration.rst @@ -39,6 +39,21 @@ With the integration you will have access to the following additional components | Query Strategies | (See :doc:`Query Strategies `) | +------------------+------------------------------------------------------------------------------------------+ +---- + +TransformerBasedClassification: Extended Functionality +====================================================== + +Layer-specific Fine-tuning +-------------------------- + +Layer-specific fine-tuning can be enabled by setting :py:class:`~small_text.integrations.transformers.classifiers.classification.FineTuningArguments` during the construction of :py:class:`~small_text.integrations.transformers.classifiers.classification.TransformerBasedClassification`. With this, you can enable layerwise gradient decay and gradual unfreezing: + +- Layerwise gradient decay: learning rates decrease the lower the layer's level is. +- Gradual unfreezing: lower layers are frozen at the start of the training and become gradually unfrozen with each epoch. + +See [HR18]_ for more details on these methods. + ----- Examples diff --git a/small_text/integrations/transformers/classifiers/classification.py b/small_text/integrations/transformers/classifiers/classification.py index 3bc969c5..0ddddf4a 100644 --- a/small_text/integrations/transformers/classifiers/classification.py +++ b/small_text/integrations/transformers/classifiers/classification.py @@ -85,7 +85,8 @@ def __init__(self, base_lr, layerwise_gradient_decay, gradual_unfreezing=-1, cut self.layerwise_gradient_decay = layerwise_gradient_decay self.gradual_unfreezing = gradual_unfreezing - self.cut_fraction = cut_fraction + # deprecated: This will be removed in the next version + _unused = cut_fraction class TransformerModelArguments(object):