feat: add LoRA/DoRA adapter layers for TimesFM 2.5 (PyTorch)
Implement production-grade PEFT adapters targeting the 2.5 architecture: - LoRALinear: low-rank A/B decomposition with scaling (alpha/rank) - DoRALinear: weight-decomposed LoRA (magnitude + direction) - inject_adapters(): freezes base weights, wraps target nn.Linear modules - Supports fused QKV (qkv_proj), attention output, and FFN layers - num_adapter_layers controls how many top layers get adapters (0=all) - target_modules selects 'all', 'attention', or 'ffn' - merge_adapters(): folds adapter deltas back into base nn.Linear - save/load_adapter_weights(): safetensors adapter-only checkpoints - PEFTConfig dataclass with all hyperparameters References: LoRA — https://arxiv.org/abs/2106.09685 DoRA — https://arxiv.org/abs/2402.09353
This commit is contained in:
@@ -0,0 +1,41 @@
|
||||
# Copyright 2025 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""PEFT (LoRA/DoRA) fine-tuning pipeline for TimesFM 2.5."""
|
||||
|
||||
from .adapters import (
|
||||
DoRALinear,
|
||||
LoRALinear,
|
||||
get_adapter_params,
|
||||
inject_adapters,
|
||||
load_adapter_weights,
|
||||
merge_adapters,
|
||||
save_adapter_weights,
|
||||
)
|
||||
from .config import PEFTConfig
|
||||
from .data import TimeSeriesDataset
|
||||
from .trainer import PEFTTrainer
|
||||
|
||||
__all__ = [
|
||||
"PEFTConfig",
|
||||
"PEFTTrainer",
|
||||
"TimeSeriesDataset",
|
||||
"LoRALinear",
|
||||
"DoRALinear",
|
||||
"inject_adapters",
|
||||
"merge_adapters",
|
||||
"save_adapter_weights",
|
||||
"load_adapter_weights",
|
||||
"get_adapter_params",
|
||||
]
|
||||
Reference in New Issue
Block a user