Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
machineuser
commited on
Commit
·
a2706d6
1
Parent(s):
2ae8da8
Sync widgets demo
Browse files
packages/tasks/src/tokenizer-data.ts
CHANGED
|
@@ -21,7 +21,7 @@ export interface AddedToken {
|
|
| 21 |
single_word?: boolean;
|
| 22 |
}
|
| 23 |
export type SpecialTokensMap = {
|
| 24 |
-
[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken;
|
| 25 |
};
|
| 26 |
/**
|
| 27 |
* Public interface for tokenizer config
|
|
|
|
| 21 |
single_word?: boolean;
|
| 22 |
}
|
| 23 |
export type SpecialTokensMap = {
|
| 24 |
+
[key in (typeof SPECIAL_TOKENS_ATTRIBUTES)[number]]?: string | AddedToken | null;
|
| 25 |
};
|
| 26 |
/**
|
| 27 |
* Public interface for tokenizer config
|
packages/widgets/src/lib/components/InferenceWidget/widgets/ConversationalWidget/ConversationalWidget.svelte
CHANGED
|
@@ -200,7 +200,7 @@
|
|
| 200 |
const value = tokenizerConfig[key];
|
| 201 |
if (typeof value === "string") {
|
| 202 |
specialTokensMap[key] = value;
|
| 203 |
-
} else if (typeof value === "object" && value.__type === "AddedToken" && value.content) {
|
| 204 |
specialTokensMap[key] = (value as AddedToken).content;
|
| 205 |
}
|
| 206 |
}
|
|
|
|
| 200 |
const value = tokenizerConfig[key];
|
| 201 |
if (typeof value === "string") {
|
| 202 |
specialTokensMap[key] = value;
|
| 203 |
+
} else if (typeof value === "object" && !!value && value.__type === "AddedToken" && value.content) {
|
| 204 |
specialTokensMap[key] = (value as AddedToken).content;
|
| 205 |
}
|
| 206 |
}
|