Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Text output for Text File and Plain Text components #1029

Merged
merged 2 commits into from
Oct 11, 2023
Merged

Text output for Text File and Plain Text components #1029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9af7775
text output for Text File and Plain Text components
jan-stanek Oct 10, 2023
4d312cb
version
jan-stanek Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 29 additions & 5 deletions packages/components/nodes/documentloaders/PlainText/PlainText.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { Document } from 'langchain/document'
import { handleEscapeCharacters } from '../../../src'

class PlainText_DocumentLoaders implements INode {
label: string
Expand All @@ -12,11 +13,12 @@ class PlainText_DocumentLoaders implements INode {
category: string
baseClasses: string[]
inputs: INodeParams[]
outputs: INodeOutputsValue[]

constructor() {
this.label = 'Plain Text'
this.name = 'plainText'
this.version = 1.0
this.version = 2.0
this.type = 'Document'
this.icon = 'plaintext.svg'
this.category = 'Document Loaders'
Expand Down Expand Up @@ -45,12 +47,25 @@ class PlainText_DocumentLoaders implements INode {
additionalParams: true
}
]
this.outputs = [
{
label: 'Document',
name: 'document',
baseClasses: this.baseClasses
},
{
label: 'Text',
name: 'text',
baseClasses: ['string', 'json']
}
]
}

async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const text = nodeData.inputs?.text as string
const metadata = nodeData.inputs?.metadata
const output = nodeData.outputs?.output as string

let alldocs: Document<Record<string, any>>[] = []

Expand All @@ -65,9 +80,9 @@ class PlainText_DocumentLoaders implements INode {
)
}

let finaldocs: Document<Record<string, any>>[] = []
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs: Document<Record<string, any>>[] = []
for (const doc of alldocs) {
const newdoc = {
...doc,
Expand All @@ -78,10 +93,19 @@ class PlainText_DocumentLoaders implements INode {
}
finaldocs.push(newdoc)
}
return finaldocs
} else {
finaldocs = alldocs
}

return alldocs
if (output === 'document') {
return finaldocs
} else {
let finaltext = ''
for (const doc of finaldocs) {
finaltext += `${doc.pageContent}\n`
}
return handleEscapeCharacters(finaltext, false)
}
}
}

Expand Down
34 changes: 30 additions & 4 deletions packages/components/nodes/documentloaders/Text/Text.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import { INode, INodeData, INodeParams } from '../../../src/Interface'
import { INode, INodeData, INodeOutputsValue, INodeParams } from '../../../src/Interface'
import { TextSplitter } from 'langchain/text_splitter'
import { TextLoader } from 'langchain/document_loaders/fs/text'
import { Document } from 'langchain/document'
import { handleEscapeCharacters } from '../../../src'

class Text_DocumentLoaders implements INode {
label: string
Expand All @@ -12,11 +14,12 @@ class Text_DocumentLoaders implements INode {
category: string
baseClasses: string[]
inputs: INodeParams[]
outputs: INodeOutputsValue[]

constructor() {
this.label = 'Text File'
this.name = 'textFile'
this.version = 1.0
this.version = 2.0
this.type = 'Document'
this.icon = 'textFile.svg'
this.category = 'Document Loaders'
Expand All @@ -43,12 +46,25 @@ class Text_DocumentLoaders implements INode {
additionalParams: true
}
]
this.outputs = [
{
label: 'Document',
name: 'document',
baseClasses: this.baseClasses
},
{
label: 'Text',
name: 'text',
baseClasses: ['string', 'json']
}
]
}

async init(nodeData: INodeData): Promise<any> {
const textSplitter = nodeData.inputs?.textSplitter as TextSplitter
const txtFileBase64 = nodeData.inputs?.txtFile as string
const metadata = nodeData.inputs?.metadata
const output = nodeData.outputs?.output as string

let alldocs = []
let files: string[] = []
Expand All @@ -75,9 +91,9 @@ class Text_DocumentLoaders implements INode {
}
}

let finaldocs: Document<Record<string, any>>[] = []
if (metadata) {
const parsedMetadata = typeof metadata === 'object' ? metadata : JSON.parse(metadata)
let finaldocs = []
for (const doc of alldocs) {
const newdoc = {
...doc,
Expand All @@ -88,9 +104,19 @@ class Text_DocumentLoaders implements INode {
}
finaldocs.push(newdoc)
}
} else {
finaldocs = alldocs
}

if (output === 'document') {
return finaldocs
} else {
let finaltext = ''
for (const doc of finaldocs) {
finaltext += `${doc.pageContent}\n`
}
return handleEscapeCharacters(finaltext, false)
}
return alldocs
}
}

Expand Down