import torch import torch.nn as nn from torchtext.datasets import AG_NEWS from torchtext.data.utils import get_tokenizer from torchtext.vocab import build_vocab_from_iterator # 数据预处理 tokenizer = get_tokenizer('basic_english') train_iter = AG_NEWS(split='train') counter = Counter() for (label, line) in train_iter: counter.update(tokenizer(line)) vocab = build_vocab_from_iterator([counter], specials=["<unk>"]) word2idx = dict(vocab.stoi) # 设定超参数 embedding_dim = 64 hidden_dim = 128 num_epochs = 10 batch_size = 64 # 定义模型 class RNN(nn.Module): def __init__(self, vocab_size, embedding_dim, hidden_dim): super(RNN, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True) self.fc = nn.Linear(hidden_dim, 4) def forward(self, x): x = self.embedding(x) out, _ = self.rnn(x) out = self.fc(out[:, -1, :]) return out # 初始化模型、优化器和损失函数 model = RNN(len(vocab), embedding_dim, hidden_dim) optimizer = torch.optim.Adam(model.parameters()) criterion = nn.CrossEntropyLoss() # 定义数据加载器 train_iter = AG_NEWS(split='train') train_data = [] for (label, line) in train_iter: label = torch.tensor([int(label)-1]) line = torch.tensor([word2idx[word] for word in tokenizer(line)]) train_data.append((line, label)) train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True) # 开始训练 for epoch in range(num_epochs): total_loss = 0.0 for input, target in train_loader: model.zero_grad() output = model(input) loss = criterion(output, target.squeeze()) loss.backward() optimizer.step() total_loss += loss.item() * input.size(0) print("Epoch: {}, Loss: {:.4f}".format(epoch+1, total_loss/len(train_data)))改错

时间: 2023-12-08 18:03:24 浏览: 319
在代码中出现了一个错误:缺少 `Counter` 的导入语句。需要在开头添加 `from collections import Counter`。 另外,在模型训练时,需要将模型设置为训练模式(`model.train()`),以启用 dropout 和 batch normalization 等功能。同时,还需要将输入和目标数据转移到 GPU 上进行计算,以加速训练过程。具体修改如下: ``` import torch import torch.nn as nn from torchtext.datasets import AG_NEWS from torchtext.data.utils import get_tokenizer from torchtext.vocab import build_vocab_from_iterator from collections import Counter # 数据预处理 tokenizer = get_tokenizer('basic_english') train_iter = AG_NEWS(split='train') counter = Counter() for (label, line) in train_iter: counter.update(tokenizer(line)) vocab = build_vocab_from_iterator([counter], specials=["<unk>"]) word2idx = dict(vocab.stoi) # 设定超参数 embedding_dim = 64 hidden_dim = 128 num_epochs = 10 batch_size = 64 # 定义模型 class RNN(nn.Module): def __init__(self, vocab_size, embedding_dim, hidden_dim): super(RNN, self).__init__() self.embedding = nn.Embedding(vocab_size, embedding_dim) self.rnn = nn.RNN(embedding_dim, hidden_dim, batch_first=True) self.fc = nn.Linear(hidden_dim, 4) def forward(self, x): x = self.embedding(x) out, _ = self.rnn(x) out = self.fc(out[:, -1, :]) return out # 初始化模型、优化器和损失函数 model = RNN(len(vocab), embedding_dim, hidden_dim) optimizer = torch.optim.Adam(model.parameters()) criterion = nn.CrossEntropyLoss() # 将模型设置为训练模式 model.train() # 将数据转移到 GPU 上 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(device) train_iter = AG_NEWS(split='train') train_data = [] for (label, line) in train_iter: label = torch.tensor([int(label)-1]).to(device) line = torch.tensor([word2idx[word] for word in tokenizer(line)]).to(device) train_data.append((line, label)) train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True) # 开始训练 for epoch in range(num_epochs): total_loss = 0.0 for input, target in train_loader: optimizer.zero_grad() output = model(input) loss = criterion(output, target.squeeze()) loss.backward() optimizer.step() total_loss += loss.item() * input.size(0) print("Epoch: {}, Loss: {:.4f}".format(epoch+1, total_loss/len(train_data))) ```
阅读全文

相关推荐

/home/wiseatc/.local/lib/python3.11/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. import pkg_resources W0703 16:30:36.069853 3914856 torch/distributed/run.py:766] W0703 16:30:36.069853 3914856 torch/distributed/run.py:766] ***************************************** W0703 16:30:36.069853 3914856 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0703 16:30:36.069853 3914856 torch/distributed/run.py:766] ***************************************** /home/wiseatc/.local/lib/python3.11/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. import pkg_resources [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,321 >> loading file tokenizer.model [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,322 >> loading file tokenizer.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,322 >> loading file added_tokens.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,322 >> loading file special_tokens_map.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,322 >> loading file tokenizer_config.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,322 >> loading file chat_template.jinja /home/wiseatc/.local/lib/python3.11/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. import pkg_resources /home/wiseatc/.local/lib/python3.11/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. import pkg_resources /home/wiseatc/.local/lib/python3.11/site-packages/jieba/_compat.py:18: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81. import pkg_resources [INFO|tokenization_utils_base.py:2313] 2025-07-03 16:30:43,904 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. [INFO|configuration_utils.py:697] 2025-07-03 16:30:43,913 >> loading configuration file /mnt/data1/models/1.5B/config.json [INFO|configuration_utils.py:771] 2025-07-03 16:30:43,919 >> Model config Qwen2Config { "_name_or_path": "/mnt/data1/models/1.5B", "architectures": [ "Qwen2ForCausalLM" ], "attention_dropout": 0.0, "bos_token_id": 151643, "eos_token_id": 151643, "hidden_act": "silu", "hidden_size": 1536, "initializer_range": 0.02, "intermediate_size": 8960, "max_position_embeddings": 131072, "max_window_layers": 21, "model_type": "qwen2", "num_attention_heads": 12, "num_hidden_layers": 28, "num_key_value_heads": 2, "rms_norm_eps": 1e-06, "rope_scaling": null, "rope_theta": 10000, "sliding_window": 4096, "tie_word_embeddings": false, "torch_dtype": "bfloat16", "transformers_version": "4.49.0", "use_cache": true, "use_mrope": false, "use_sliding_window": false, "vocab_size": 151936 } [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file tokenizer.model [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file tokenizer.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file added_tokens.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file special_tokens_map.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file tokenizer_config.json [INFO|tokenization_utils_base.py:2048] 2025-07-03 16:30:43,920 >> loading file chat_template.jinja [INFO|tokenization_utils_base.py:2313] 2025-07-03 16:30:44,493 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained. /usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via init_process_group or barrier . Using the current device set by the user. warnings.warn( # warn only once [rank1]:[W703 16:30:45.102845887 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. /usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via init_process_group or barrier . Using the current device set by the user. warnings.warn( # warn only once [rank2]:[W703 16:30:45.126706430 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. /usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via init_process_group or barrier . Using the current device set by the user. warnings.warn( # warn only once [rank3]:[W703 16:30:45.136836682 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. Setting num_proc from 16 back to 1 for the train split to disable multiprocessing as it only contains one shard. Generating train split: 0 examples [00:00, ? examples/s] Generating train split: 120 examples [00:00, 6525.39 examples/s] Converting format of dataset (num_proc=16): 0%| | 0/120 [00:00<?, ? examples/s] Converting format of dataset (num_proc=16): 0%| | 0/120 [00:00<?, ? examples/s] Converting format of dataset (num_proc=16): 0%| | 0/120 [00:00<?, ? examples/s] /usr/local/lib/python3.11/dist-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via init_process_group or barrier . Using the current device set by the user. warnings.warn( # warn only once [rank0]:[W703 16:31:05.679961201 ProcessGroupNCCL.cpp:4718] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can pecify device_id in init_process_group() to force use of a particular device. [rank0]: multiprocess.pool.RemoteTraceback: [rank0]: """ [rank0]: Traceback (most recent call last): [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/multiprocess/pool.py", line 125, in worker [rank0]: result = (True, func(*args, **kwds)) [rank0]: ^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 688, in _write_generator_to_queue [rank0]: for i, result in enumerate(func(**kwargs)): [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3501, in _map_single [rank0]: for i, example in iter_outputs(shard_iterable): [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3475, in iter_outputs [rank0]: yield i, apply_function(example, i, offset=offset) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3398, in apply_function [rank0]: processed_inputs = function(*fn_args, *additional_args, **fn_kwargs) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/data/converter.py", line 94, in __call__ [rank0]: if self.dataset_attr.prompt and example[self.dataset_attr.prompt]: [rank0]: ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/formatting/formatting.py", line 278, in __getitem__ [rank0]: value = self.data[key] [rank0]: ~~~~~~~~~^^^^^ [rank0]: KeyError: 'instruction' [rank0]: """ [rank0]: The above exception was the direct cause of the following exception: [rank0]: Traceback (most recent call last): [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/launcher.py", line 23, in <module> [rank0]: launch() [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/launcher.py", line 19, in launch [rank0]: run_exp() [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/train/tuner.py", line 110, in run_exp [rank0]: _training_function(config={"args": args, "callbacks": callbacks}) [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/train/tuner.py", line 72, in _training_function [rank0]: run_sft(model_args, data_args, training_args, finetuning_args, generating_args, callbacks) [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/train/sft/workflow.py", line 51, in run_sft [rank0]: dataset_module = get_dataset(template, model_args, data_args, training_args, stage="sft", **tokenizer_module) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/data/loader.py", line 304, in get_dataset [rank0]: dataset = _get_merged_dataset(data_args.dataset, model_args, data_args, training_args, stage) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/data/loader.py", line 182, in _get_merged_dataset [rank0]: datasets[dataset_name] = _load_single_dataset(dataset_attr, model_args, data_args, training_args) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/data/loader.py", line 162, in _load_single_dataset [rank0]: return align_dataset(dataset, dataset_attr, data_args, training_args) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/LLaMA-Factory/src/llamafactory/data/converter.py", line 279, in align_dataset [rank0]: return dataset.map( [rank0]: ^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 557, in wrapper [rank0]: out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/arrow_dataset.py", line 3171, in map [rank0]: for rank, done, content in iflatmap_unordered( [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 728, in iflatmap_unordered [rank0]: [async_result.get(timeout=0.05) for async_result in async_results] [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/datasets/utils/py_utils.py", line 728, in [rank0]: [async_result.get(timeout=0.05) for async_result in async_results] [rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank0]: File "/home/wiseatc/.local/lib/python3.11/site-packages/multiprocess/pool.py", line 774, in get [rank0]: raise self._value [rank0]: KeyError: 'instruction' [rank0]:[W703 16:31:06.912491219 ProcessGroupNCCL.cpp:1479] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) W0703 16:31:07.960560 3914856 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 3914916 closing signal SIGTERM W0703 16:31:07.961188 3914856 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 3914917 closing signal SIGTERM W0703 16:31:07.961536 3914856 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 3914918 closing signal SIGTERM E0703 16:31:08.371267 3914856 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 1) local_rank: 0 (pid: 3914915) of binary: /usr/bin/python3.11 Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in <module> sys.exit(main()) ^^^^^^ File "/usr/local/lib/python3.11/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper return f(*args, **kwargs) ^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/dist-packages/torch/distributed/run.py", line 892, in main run(args) File "/usr/local/lib/python3.11/dist-packages/torch/distributed/run.py", line 883, in run elastic_launch( File "/usr/local/lib/python3.11/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ return launch_agent(self._config, self._entrypoint, list(args)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/local/lib/python3.11/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent raise ChildFailedError( torch.distributed.elastic.multiprocessing.errors.ChildFailedError: ============================================================ /home/wiseatc/LLaMA-Factory/src/llamafactory/launcher.py FAILED ------------------------------------------------------------ Failures: <NO_OTHER_FAILURES> ------------------------------------------------------------ Root Cause (first observed failure): [0]: time : 2025-07-03_16:31:07 host : wiseatc-Super-Server rank : 0 (local_rank: 0) exitcode : 1 (pid: 3914915) error_file: <N/A> traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html ============================================================ Traceback (most recent call last): File "/home/wiseatc/.local/bin/llamafactory-cli", line 8, in <module> sys.exit(main()) ^^^^^^ File "/home/wiseatc/LLaMA-Factory/src/llamafactory/cli.py", line 130, in main process = subprocess.run( ^^^^^^^^^^^^^^^ File "/usr/lib/python3.11/subprocess.py", line 569, in run raise CalledProcessError(retcode, process.args, subprocess.CalledProcessError: Command '['torchrun', '--nnodes', '1', '--node_rank', '0', '--nproc_per_node', '4', '--master_addr', '127.0.0.1', '--master_port', '41919', '/home/wiseatc/LLaMA-Factory/src/llamafactory/launcher.py', 'saves/DeepSeek-R1-1.5B-Distill/lora/train_2025-07-03-16-29-46/training_args.yaml']' returned non-zero exit status 1.

我已经下载了tiktoken和protobuf库,D:\PythonProject\deepseekai.venv\Scripts\python.exe D:\PythonProject\deepseekai\train_weather_model.py PyTorch 版本: 2.3.1+cu118 CUDA 可用: True GPU 名称: NVIDIA GeForce GTX 1650 Ti You are using the default legacy behaviour of the <class ‘transformers.models.llama.tokenization_llama_fast.LlamaTokenizerFast’>. This is expected, and simply means that the legacy (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set legacy=False. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message. Traceback (most recent call last): File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\convert_slow_tokenizer.py”, line 1737, in convert_slow_tokenizer ).converted() ^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\convert_slow_tokenizer.py”, line 1631, in converted tokenizer = self.tokenizer() ^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\convert_slow_tokenizer.py”, line 1624, in tokenizer vocab_scores, merges = self.extract_vocab_merges_from_model(self.vocab_file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\convert_slow_tokenizer.py”, line 1600, in extract_vocab_merges_from_model bpe_ranks = load_tiktoken_bpe(tiktoken_url) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\tiktoken\load.py”, line 148, in load_tiktoken_bpe contents = read_file_cached(tiktoken_bpe_file, expected_hash) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\tiktoken\load.py”, line 48, in read_file_cached cache_key = hashlib.sha1(blobpath.encode()).hexdigest() ^^^^^^^^^^^^^^^ AttributeError: ‘NoneType’ object has no attribute ‘encode’ During handling of the above exception, another exception occurred: Traceback (most recent call last): File “D:\PythonProject\deepseekai\train_weather_model.py”, line 31, in <module> tokenizer = AutoTokenizer.from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\models\auto\tokenization_auto.py”, line 1032, in from_pretrained return tokenizer_class_fast.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\tokenization_utils_base.py”, line 2025, in from_pretrained return cls._from_pretrained( ^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\tokenization_utils_base.py”, line 2278, in _from_pretrained tokenizer = cls(*init_inputs, **init_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\models\llama\tokenization_llama_fast.py”, line 154, in init super().init( File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\tokenization_utils_fast.py”, line 139, in init fast_tokenizer = convert_slow_tokenizer(self, from_tiktoken=True) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File “D:\PythonProject\deepseekai.venv\Lib\site-packages\transformers\convert_slow_tokenizer.py”, line 1739, in convert_slow_tokenizer raise ValueError( ValueError: Converting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: [‘AlbertTokenizer’, ‘BartTokenizer’, ‘BarthezTokenizer’, ‘BertTokenizer’, ‘BigBirdTokenizer’, ‘BlenderbotTokenizer’, ‘CamembertTokenizer’, ‘CLIPTokenizer’, ‘CodeGenTokenizer’, ‘ConvBertTokenizer’, ‘DebertaTokenizer’, ‘DebertaV2Tokenizer’, ‘DistilBertTokenizer’, ‘DPRReaderTokenizer’, ‘DPRQuestionEncoderTokenizer’, ‘DPRContextEncoderTokenizer’, ‘ElectraTokenizer’, ‘FNetTokenizer’, ‘FunnelTokenizer’, ‘GPT2Tokenizer’, ‘HerbertTokenizer’, ‘LayoutLMTokenizer’, ‘LayoutLMv2Tokenizer’, ‘LayoutLMv3Tokenizer’, ‘LayoutXLMTokenizer’, ‘LongformerTokenizer’, ‘LEDTokenizer’, ‘LxmertTokenizer’, ‘MarkupLMTokenizer’, ‘MBartTokenizer’, ‘MBart50Tokenizer’, ‘MPNetTokenizer’, ‘MobileBertTokenizer’, ‘MvpTokenizer’, ‘NllbTokenizer’, ‘OpenAIGPTTokenizer’, ‘PegasusTokenizer’, ‘Qwen2Tokenizer’, ‘RealmTokenizer’, ‘ReformerTokenizer’, ‘RemBertTokenizer’, ‘RetriBertTokenizer’, ‘RobertaTokenizer’, ‘RoFormerTokenizer’, ‘SeamlessM4TTokenizer’, ‘SqueezeBertTokenizer’, ‘T5Tokenizer’, ‘UdopTokenizer’, ‘WhisperTokenizer’, ‘XLMRobertaTokenizer’, ‘XLNetTokenizer’, ‘SplinterTokenizer’, ‘XGLMTokenizer’, ‘LlamaTokenizer’, ‘CodeLlamaTokenizer’, ‘GemmaTokenizer’, ‘Phi3Tokenizer’] Process finished with exit code 1

D:\PythonProject\deepseekai\.venv\Scripts\python.exe D:\PythonProject\deepseekai\train_weather_model.py 模型文件已复制到: ./local-deepseek-model\model.safetensors 配置文件已创建: config.json 分词器配置文件已创建: tokenizer_config.json You are using the default legacy behaviour of the <class 'transformers.models.llama.tokenization_llama.LlamaTokenizer'>. This is expected, and simply means that the legacy (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set legacy=False. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file you can ignore this message Traceback (most recent call last): File "D:\PythonProject\deepseekai\train_weather_model.py", line 68, in <module> tokenizer = AutoTokenizer.from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\models\auto\tokenization_auto.py", line 1013, in from_pretrained return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\tokenization_utils_base.py", line 2025, in from_pretrained return cls._from_pretrained( ^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\tokenization_utils_base.py", line 2063, in _from_pretrained slow_tokenizer = (cls.slow_tokenizer_class)._from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\tokenization_utils_base.py", line 2278, in _from_pretrained tokenizer = cls(*init_inputs, **init_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\models\llama\tokenization_llama.py", line 171, in __init__ self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\models\llama\tokenization_llama.py", line 198, in get_spm_processor tokenizer.Load(self.vocab_file) File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\sentencepiece\__init__.py", line 961, in Load return self.LoadFromFile(model_file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\sentencepiece\__init__.py", line 316, in LoadFromFile return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: not a string Process finished with exit code 1

D:\PythonProject\deepseekai\.venv\Scripts\python.exe D:\PythonProject\deepseekai\train_weather_model.py sentencepiece 已安装: 0.2.0 创建/下载: tokenizer.model 下载失败: 401 Client Error: Unauthorized for url: https://huggingface.co/deepseek-ai/deepseek-llm-1.3b-base/resolve/main/tokenizer.model 无法下载 tokenizer.model, 请手动下载 Traceback (most recent call last): File "D:\PythonProject\deepseekai\train_weather_model.py", line 134, in <module> tokenizer = LlamaTokenizer.from_pretrained( ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\tokenization_utils_base.py", line 2025, in from_pretrained return cls._from_pretrained( ^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\tokenization_utils_base.py", line 2278, in _from_pretrained tokenizer = cls(*init_inputs, **init_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\models\llama\tokenization_llama.py", line 171, in __init__ self.sp_model = self.get_spm_processor(kwargs.pop("from_slow", False)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\transformers\models\llama\tokenization_llama.py", line 198, in get_spm_processor tokenizer.Load(self.vocab_file) File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\sentencepiece\__init__.py", line 961, in Load return self.LoadFromFile(model_file) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "D:\PythonProject\deepseekai\.venv\Lib\site-packages\sentencepiece\__init__.py", line 316, in LoadFromFile return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: not a string Process finished with exit code 1

最新推荐

recommend-type

基于单片机的水位自动检测与控制系统开题报告.doc

基于单片机的水位自动检测与控制系统开题报告.doc
recommend-type

cc65 Windows完整版发布:6502 C开发工具

cc65是一个针对6502处理器的完整C编程开发环境,特别适用于Windows操作系统。6502处理器是一种经典的8位微处理器,于1970年代被广泛应用于诸如Apple II、Atari 2600、NES(任天堂娱乐系统)等早期计算机和游戏机中。cc65工具集能够允许开发者使用C语言编写程序,这对于那些希望为这些老旧系统开发软件的程序员来说是一大福音,因为相较于汇编语言,C语言更加高级、易读,并且具备更好的可移植性。 cc65开发工具包主要包含以下几个重要组件: 1. C编译器:这是cc65的核心部分,它能够将C语言源代码编译成6502处理器的机器码。这使得开发者可以用高级语言编写程序,而不必处理低级的汇编指令。 2. 链接器:链接器负责将编译器生成的目标代码和库文件组合成一个单独的可执行程序。在6502的开发环境中,链接器还需要处理各种内存段的定位和映射问题。 3. 汇编器:虽然主要通过C语言进行开发,但某些底层操作仍然可能需要使用汇编语言来实现。cc65包含了一个汇编器,允许程序员编写汇编代码段。 4. 库和运行时:cc65提供了一套标准库,这些库函数为C语言提供了支持,并且对于操作系统级别的功能进行了封装,使得开发者能够更方便地进行编程。运行时支持包括启动代码、中断处理、内存管理等。 5. 开发工具和文档:除了基本的编译、链接和汇编工具外,cc65还提供了一系列辅助工具,如反汇编器、二进制文件编辑器、交叉引用器等。同时,cc65还包含丰富的文档资源,为开发者提供了详尽的使用指南、编程参考和示例代码。 cc65可以广泛用于学习和开发6502架构相关的软件,尤其适合那些对6502处理器、复古计算机或者早期游戏系统有兴趣的开发者。这些开发者可能想要创建或修改旧式游戏、系统软件、仿真器,或者进行技术研究和学习。 尽管cc65是一个功能强大的工具,但它也要求开发者对目标平台的硬件架构和操作系统有足够的了解。这是因为6502并非现代处理器,它对内存访问、I/O操作和中断管理等有着特殊的限制和要求。因此,使用cc65需要开发者具备一定的背景知识,包括但不限于6502指令集、内存映射、硬件寄存器操作等方面的内容。 此外,cc65针对Windows平台进行了优化和封装,使得它可以在Windows操作系统上无缝运行,这对于习惯于Windows环境的用户是一个好消息。不过,如果用户使用的是其他操作系统,可能需要通过相应的兼容层或虚拟机来运行Windows环境,以便使用cc65工具。 总的来说,cc65提供了一种相对简单的方式来开发运行在6502处理器上的软件。它极大地降低了开发难度,使得更多的人能够接触和参与到基于6502平台的软件开发中来,为这个历史悠久的平台注入了新的活力。
recommend-type

【CLIP模型实战】:从数据预处理到代码实现的图文相似度计算完全指南

# 1. CLIP模型概述 在人工智能领域中,CLIP(Contrastive Language-Image Pre-training)模型作为一款开创性的多模态学习技术,已经成为业界瞩目的焦点。CLIP模型是由OpenAI在2021年推出的,其创新之处在于通过大规模的图文对齐数据进行预训练,能够理解和匹配图像与自然语言描述之间的关联性。CLIP突破了传统方法在特定任务上需要大规模标注数据的限制
recommend-type

车载以太网doip协议格式

<think>首先,用户的问题是:“我想查找关于车载以太网DOIP协议的数据格式或规范”。用户引用了站内内容,其中提到DoIP是基于以太网的通讯协议,用于传输UDS协议的数据,规范于ISO13400标准。关键点:-DoIP协议:DiagnosticcommunicationoverInternetProtocol-规范:ISO13400标准-数据格式:我需要提供关于DoIP数据格式的详细信息根据系统级指令:-所有行内数学表达式使用$...$格式-独立公式使用$$...$$格式并单独成段-LaTeX语法正确-使用中文回答-生成相关问题-回答中引用的段落末尾自然地添加引用标识-回答结构清晰,帮助用
recommend-type

JavaScript中文帮助手册:初学者实用指南

### JavaScript中文帮助手册知识点概述 #### 1. JavaScript简介 JavaScript是一种轻量级的编程语言,广泛用于网页开发。它能够增强用户与网页的交互性,使得网页内容变得动态和富有生气。JavaScript能够操纵网页中的HTML元素,响应用户事件,以及与后端服务器进行通信等。 #### 2. JavaScript基本语法 JavaScript的语法受到了Java和C语言的影响,包括变量声明、数据类型、运算符、控制语句等基础组成部分。以下为JavaScript中常见的基础知识点: - 变量:使用关键字`var`、`let`或`const`来声明变量,其中`let`和`const`是ES6新增的关键字,提供了块级作用域和不可变变量的概念。 - 数据类型:包括基本数据类型(字符串、数值、布尔、null和undefined)和复合数据类型(对象、数组和函数)。 - 运算符:包括算术运算符、关系运算符、逻辑运算符、位运算符等。 - 控制语句:条件判断语句(if...else、switch)、循环语句(for、while、do...while)等。 - 函数:是JavaScript中的基础,可以被看作是一段代码的集合,用于封装重复使用的代码逻辑。 #### 3. DOM操作 文档对象模型(DOM)是HTML和XML文档的编程接口。JavaScript可以通过DOM操作来读取、修改、添加或删除网页中的元素和内容。以下为DOM操作的基础知识点: - 获取元素:使用`getElementById()`、`getElementsByTagName()`等方法获取页面中的元素。 - 创建和添加元素:使用`document.createElement()`创建新元素,使用`appendChild()`或`insertBefore()`方法将元素添加到文档中。 - 修改和删除元素:通过访问元素的属性和方法,例如`innerHTML`、`textContent`、`removeChild()`等来修改或删除元素。 - 事件处理:为元素添加事件监听器,响应用户的点击、鼠标移动、键盘输入等行为。 #### 4. BOM操作 浏览器对象模型(BOM)提供了独立于内容而与浏览器窗口进行交互的对象和方法。以下是BOM操作的基础知识点: - window对象:代表了浏览器窗口本身,提供了许多属性和方法,如窗口大小调整、滚动、弹窗等。 - location对象:提供了当前URL信息的接口,可以用来获取URL、重定向页面等。 - history对象:提供了浏览器会话历史的接口,可以进行导航历史操作。 - screen对象:提供了屏幕信息的接口,包括屏幕的宽度、高度等。 #### 5. JavaScript事件 JavaScript事件是用户或浏览器自身执行的某些行为,如点击、页面加载、键盘按键、鼠标移动等。通过事件,JavaScript可以对这些行为进行响应。以下为事件处理的基础知识点: - 事件类型:包括鼠标事件、键盘事件、表单事件、窗口事件等。 - 事件监听:通过`addEventListener()`方法为元素添加事件监听器,规定当事件发生时所要执行的函数。 - 事件冒泡:事件从最深的节点开始,然后逐级向上传播到根节点。 - 事件捕获:事件从根节点开始,然后逐级向下传播到最深的节点。 #### 6. JavaScript高级特性 随着ECMAScript标准的演进,JavaScript引入了许多高级特性,这些特性包括但不限于: - 对象字面量增强:属性简写、方法简写、计算属性名等。 - 解构赋值:可以从数组或对象中提取数据,赋值给变量。 - 模板字符串:允许嵌入表达式。 - 异步编程:Promise、async/await等用于处理异步操作。 - 模块化:使用`import`和`export`关键字导入和导出模块。 - 类和模块:引入了`class`关键字,允许使用面向对象编程风格定义类,以及模块的声明。 #### 7. 开发工具和调试技巧 为了提高JavaScript开发效率和调试问题,以下是一些常用的工具和调试技巧: - 浏览器的开发者工具:包括控制台(Console)、元素查看器(Elements)、网络监控(Network)、源码编辑器(Sources)等。 - 断点调试:在源码编辑器中设置断点,逐步执行代码,查看变量值和程序流程。 - console.log:在控制台输出日志,帮助理解程序执行流程和变量状态。 - 使用JavaScript验证工具:如JSHint、ESLint等,可以在开发过程中进行代码质量检查。 以上就是《JavaScript中文帮助手册》中可能包含的主要知识点。作为初学者,通过这些内容可以系统地学习和掌握JavaScript基础和进阶知识,实现从初学到实践的跨越。在实际应用中,还需结合具体实例和项目练习,不断加深理解和熟练操作。
recommend-type

深入理解MySQL存储引擎:InnoDB与MyISAM的终极对决

# 1. MySQL存储引擎概述 MySQL数据库的灵活性和高性能在很大程度上得益于其存储引擎架构。**存储引擎**是MySQL中用于存储、索引、查询数据的底层软件模块。不同的存储引擎拥有不同的功能和特性,允许数据库管理员针对特定的应用需求选择最佳的存储引擎。例如,**InnoDB**提供事务支持和行级锁定,适用于需要ACID(原子
recommend-type

window中系统中断,cpu占用100%

<think>首先,用户的问题是关于解决Windows系统中“系统中断”导致CPU占用100%的问题。我需要基于提供的引用和系统指令来构建一个结构清晰的回答。系统指令:-所有行内数学表达式必须使用$...$格式。-独立公式使用$$...$$格式并单独成段。-LaTeX语法正确。-使用中文回答。-生成相关问题。-在回答中引用的段落末尾添加引用标识,例如[^1]。-回答结构清晰,帮助用户逐步解决问题。-参考站内引用:引用[1]是关于CPU使用率100%的应急措施,引用[2]是关于Java程序CPU占用过高的解决步骤,引用[3]是关于CPU软中断导致高利用率的分析与解决方案。用户的问题:Window
recommend-type

C++Builder6.0缺失帮助文件的解决方案

标题“BCB6.0帮助文件”和描述“很多用户的C++Builder6.0的版本没有帮助文件,help文件对学习和研究BCB6.0是很重要的。”表明了我们讨论的主题是关于C++Builder(通常简称BCB)6.0版本的官方帮助文件。C++Builder是一款由Borland公司(后被Embarcadero Technologies公司收购)开发的集成开发环境(IDE),专门用于C++语言的开发。该软件的第六版,即BCB6.0,于2002年发布,是该系列的一个重要版本。在这个版本中,提供了一个帮助文件,对于学习和研究BCB6.0至关重要。因为帮助文件中包含了大量关于IDE使用的指导、编程API的参考、示例代码等,是使用该IDE不可或缺的资料。 我们可以通过【压缩包子文件的文件名称列表】中的“BCB6.0_Help”推测,这可能是一个压缩文件,包含了帮助文件的副本,可能是一个ZIP或者其他格式的压缩文件。该文件的名称“BCB6.0_Help”暗示了文件中包含的是与C++Builder6.0相关的帮助文档。在实际获取和解压该文件后,用户能够访问到详尽的文档,以便更深入地了解和利用BCB6.0的功能。 BCB6.0帮助文件的知识点主要包括以下几个方面: 1. 环境搭建和配置指南:帮助文档会解释如何安装和配置BCB6.0环境,包括如何设置编译器、调试器和其他工具选项,确保用户能够顺利开始项目。 2. IDE使用教程:文档中应包含有关如何操作IDE界面的说明,例如窗口布局、菜单结构、快捷键使用等,帮助用户熟悉开发环境。 3. 语言参考:C++Builder6.0支持C++语言,因此帮助文件会包含C++语言核心特性的说明、标准库参考、模板和STL等。 4. VCL框架说明:BCB6.0是基于Visual Component Library(VCL)框架的,帮助文件会介绍如何使用VCL构建GUI应用程序,包括组件的使用方法、事件处理、窗体设计等。 5. 数据库编程:文档会提供关于如何利用C++Builder进行数据库开发的指导,涵盖了数据库连接、SQL语言、数据集操作等关键知识点。 6. 高级功能介绍:帮助文件还会介绍一些高级功能,如使用组件面板、定制组件、深入到编译器优化、代码分析工具的使用等。 7. 示例项目和代码:为了更好地演示如何使用IDE和语言特性,帮助文件通常包含了一个或多个示例项目以及一些实用的代码片段。 8. 第三方插件和工具:BCB6.0还可能支持第三方插件,帮助文件可能会对一些广泛使用的插件进行介绍和解释如何安装和使用它们。 9. 故障排除和调试:文档会提供一些常见问题的解决方案、调试技巧以及性能调优建议。 10. 版本更新记录:虽然版本更新记录通常不会在帮助文件内详细描述,但可能会提到重大的新增特性、改进和已知问题。 11. 联系方式和资源:帮助文件中可能会包含Embarcadero公司的技术支持联系方式,以及推荐的外部资源,比如论坛、在线文档链接和社区。 在学习和研究BCB6.0时,帮助文件是一个十分宝贵的资源,它能提供快速准确的信息和指导。对于任何一个使用BCB6.0进行开发的程序员来说,熟悉这些知识点是必不可少的。
recommend-type

【湖北专升本MySQL强化训练】:5大SQL语句编写技巧,迅速提升实战能力

# 1. SQL语言基础 ## 1.1 SQL语言简介 SQL,即结构化查询语言(Structured Query Language),是一种用于管理关系数据库管理系统(RDBMS)的标准编程语言。它广泛应用于数据的查询、插入、更新和删除操作,是数据库管理员和开发人员必须掌握的核心技能。 ## 1.2 SQL语言的组成 SQL语言主要由数据定义语言(DDL),数据操纵语言
recommend-type

HFSS如何设置网格化细化

<think>我们正在讨论HFSS中的网格细化设置。根据引用[2]和[3],HFSS使用自适应网格剖分技术,通过迭代细化网格直到满足收敛标准。用户想了解具体的设置方法。关键点:1.自适应网格剖分过程:HFSS通过比较连续迭代之间的S参数(或其他参数)的变化来判断收敛。2.收敛标准设置:用户可以设置收敛的阈值(如ΔS)。3.初始网格设置:在求解设置中,可以设置初始网格的选项(如Lambda细化)。步骤:1.在HFSS中,右键点击工程树中的“Analysis”节点,选择“AddSolutionSetup”来添加一个新的求解设置(或者编辑已有的求解设置)。2.在求解设置对话框中,切换到“Genera