From 5058900e53609b0c7f8dc58ae10d9a03f2ef9f2b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:30:36 +0000 Subject: [PATCH 1/2] [pre-commit.ci] pre-commit autoupdate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v5.0.0 → v6.0.0](https://github.com/pre-commit/pre-commit-hooks/compare/v5.0.0...v6.0.0) - [github.com/astral-sh/ruff-pre-commit: v0.11.0 → v0.16.0](https://github.com/astral-sh/ruff-pre-commit/compare/v0.11.0...v0.16.0) --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b4f4000..11bdd97 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.0 + rev: v0.16.0 hooks: - id: ruff args: [--fix] From 90e5732bf8d28a1545a438cf764bc4b71d6c4d70 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 Jul 2026 16:30:50 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- data/test_data_loading.py | 3 ++- dataset.py | 11 ++++++----- demo.py | 5 +++-- full_automation.py | 5 ++--- utils/tool_utils.py | 6 +++--- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/data/test_data_loading.py b/data/test_data_loading.py index ad481da..6749762 100644 --- a/data/test_data_loading.py +++ b/data/test_data_loading.py @@ -1,6 +1,7 @@ -import pytest import json +import pytest + @pytest.fixture def load_data(): diff --git a/dataset.py b/dataset.py index c717595..675204e 100644 --- a/dataset.py +++ b/dataset.py @@ -1,9 +1,10 @@ import json -from typing import Any, Dict, List +from typing import Any import torch from loguru import logger from torch.utils.data import Dataset + from utils.tool_utils import function_formatter @@ -18,10 +19,10 @@ def __init__(self, file, tokenizer, max_seq_length, template): self.observation_format = template["observation_format"] self.max_seq_length = max_seq_length - logger.info("Loading data: {}".format(file)) + logger.info(f"Loading data: {file}") with open(file, "r", encoding="utf8") as f: data_list = f.readlines() - logger.info("There are {} data in dataset".format(len(data_list))) + logger.info(f"There are {len(data_list)} data in dataset") self.data_list = data_list def __len__(self): @@ -93,13 +94,13 @@ def __getitem__(self, index): return inputs -class SFTDataCollator(object): +class SFTDataCollator: def __init__(self, tokenizer, max_seq_length): self.tokenizer = tokenizer self.max_seq_length = max_seq_length self.pad_token_id = tokenizer.pad_token_id - def __call__(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]: + def __call__(self, batch: list[dict[str, Any]]) -> dict[str, Any]: # Find the maximum length in the batch lengths = [len(x["input_ids"]) for x in batch if x["input_ids"] is not None] # Take the maximum length in the batch, if it exceeds max_seq_length, take max_seq_length diff --git a/demo.py b/demo.py index 1c6ee73..6187472 100644 --- a/demo.py +++ b/demo.py @@ -2,10 +2,11 @@ from dataclasses import dataclass import torch +from datasets import Dataset from peft import LoraConfig from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig -from trl import SFTTrainer, SFTConfig -from datasets import Dataset +from trl import SFTConfig, SFTTrainer + from dataset import SFTDataCollator, SFTDataset from utils.constants import model2template diff --git a/full_automation.py b/full_automation.py index d1efd5e..5dfdde6 100644 --- a/full_automation.py +++ b/full_automation.py @@ -3,8 +3,8 @@ import requests import yaml -from loguru import logger from huggingface_hub import HfApi +from loguru import logger from demo import LoraTrainingArguments, train_lora from utils.constants import model2base_model, model2size @@ -36,8 +36,7 @@ # download in chunks response = requests.get(data_url, stream=True) with open("data/demo_data.jsonl", "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) + f.writelines(response.iter_content(chunk_size=8192)) # train all feasible models and merge for model_id in all_training_args.keys(): diff --git a/utils/tool_utils.py b/utils/tool_utils.py index 2aa2934..90da8d0 100644 --- a/utils/tool_utils.py +++ b/utils/tool_utils.py @@ -1,5 +1,5 @@ -from typing import Dict, Any, List, Tuple import json +from typing import Any DEFAULT_TOOL_PROMPT = ( "You have access to the following tools:\n{tool_text}" @@ -14,7 +14,7 @@ DEFAULT_FUNCTION_SLOTS = "Action: {name}\nAction Input: {arguments}\n" -def tool_formater(tools: List[Dict[str, Any]]) -> str: +def tool_formater(tools: list[dict[str, Any]]) -> str: tool_text = "" tool_names = [] for tool in tools: @@ -52,7 +52,7 @@ def tool_formater(tools: List[Dict[str, Any]]) -> str: def function_formatter(tool_calls, function_slots=DEFAULT_FUNCTION_SLOTS) -> str: - functions: List[Tuple[str, str]] = [] + functions: list[tuple[str, str]] = [] if not isinstance(tool_calls, list): tool_calls = [tool_calls] # parrallel function calls