From 3ba8daab0ca35f8481d557a42934ec01ffd172b7 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Sun, 18 Jan 2026 14:21:09 +0100 Subject: [PATCH] Work on AI doc parsing protopype --- htdocs/ai/admin/custom_prompt.php | 10 +- htdocs/ai/class/ai.class.php | 167 ++++++--- htdocs/ai/lib/ai.lib.php | 8 +- htdocs/core/actions_linkedfiles.inc.php | 2 + htdocs/core/ajax/ajaxstatusprospect.php | 3 +- htdocs/core/ajax/ajaxuploadpage.php | 444 ++++++++++++++++++++++++ htdocs/core/upload_page.php | 20 +- htdocs/core/upload_page2.php | 119 ------- htdocs/langs/en_US/admin.lang | 5 +- htdocs/langs/en_US/exports.lang | 1 + 10 files changed, 597 insertions(+), 182 deletions(-) create mode 100644 htdocs/core/ajax/ajaxuploadpage.php delete mode 100644 htdocs/core/upload_page2.php diff --git a/htdocs/ai/admin/custom_prompt.php b/htdocs/ai/admin/custom_prompt.php index cf6664c6690..40158f03c6b 100644 --- a/htdocs/ai/admin/custom_prompt.php +++ b/htdocs/ai/admin/custom_prompt.php @@ -26,10 +26,6 @@ // Load Dolibarr environment require '../../main.inc.php'; -require_once DOL_DOCUMENT_ROOT."/core/lib/admin.lib.php"; -require_once DOL_DOCUMENT_ROOT."/ai/lib/ai.lib.php"; -require_once DOL_DOCUMENT_ROOT."/core/class/html.formai.class.php"; - /** * @var Conf $conf * @var DoliDB $db @@ -37,6 +33,9 @@ require_once DOL_DOCUMENT_ROOT."/core/class/html.formai.class.php"; * @var Translate $langs * @var User $user */ +require_once DOL_DOCUMENT_ROOT."/core/lib/admin.lib.php"; +require_once DOL_DOCUMENT_ROOT."/ai/lib/ai.lib.php"; +require_once DOL_DOCUMENT_ROOT."/core/class/html.formai.class.php"; $langs->loadLangs(array("admin", "website", "other")); @@ -432,9 +431,10 @@ if ($action == 'edit' || $action == 'create' || $action == 'deleteproperty') { print '
'; } - +// Custom models if ($action == 'edit' || $action == 'create' || $action == 'deleteproperty') { print load_fiche_titre($langs->trans("AIModelForFeature", $arrayofai[$aiservice]['label']), '', ''); + print $formSetup->generateOutput(true); } diff --git a/htdocs/ai/class/ai.class.php b/htdocs/ai/class/ai.class.php index 4c93e825fed..7439737ab1c 100644 --- a/htdocs/ai/class/ai.class.php +++ b/htdocs/ai/class/ai.class.php @@ -60,6 +60,8 @@ class Ai const AI_DEFAULT_PROMPT_FOR_TEXT_SUMMARIZE = 'You are a writer, make the answer in the same language than the original text to summarize.'; const AI_DEFAULT_PROMPT_FOR_TEXT_REPHRASER = 'You are a writer, give only one answer with no comment and explanation and give the answer in the same language than the original text to rephrase.'; const AI_DEFAULT_PROMPT_FOR_EXTRAFIELD_FILLER = 'Give only one answer with no comment and explanation, I want the text to be ready to copy and paste.'; + const AI_DEFAULT_PROMPT_FOR_DOC_PARSING = 'You are an assistant to anayze documents.'; + /** * Constructor @@ -76,16 +78,28 @@ class Ai $this->apiKey = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_KEY'); } + /** + * get API Service + * + * @return string API service + */ + public function getApiService() + { + return $this->apiService; + } + /** * Generate response of instructions * - * @param string $instructions Instruction to generate content - * @param string $model Model name ('gpt-4.1-turbo', 'gpt-4.1', 'dall-e-3', ...) - * @param string $function Code of the feature we want to use ('textgeneration', 'transcription', 'audiogeneration', 'imagegeneration', 'translation') - * @param string $format Format for output ('', 'html', ...) + * @param string|array $instructions String instruction to generate content (or file path) or array of payload or ID of file with function threads + * @param string $model Model name ('gpt-4.1-turbo', 'gpt-4.1', 'dall-e-3', ...) + * @param string $function Code of the feature we want to use ('textgeneration', 'transcription', 'audiogeneration', 'imagegeneration', 'translation', 'docparsing') + * @param string $format Format for output ('', 'html', ...) + * @param array $moreheaders More headers + * @param string $moreendpoint Add a part to endpoint url * @return string|array{error:bool,message:string,code?:int,curl_error_no?:int,format?:string,service?:string,function?:string} $response Text or array if error */ - public function generateContent($instructions, $model = 'auto', $function = 'textgeneration', $format = '') + public function generateContent($instructions, $model = 'auto', $function = 'textgeneration', $format = '', $moreheaders = array(), $moreendpoint = '') { global $dolibarr_main_data_root; @@ -114,17 +128,34 @@ class Ai } elseif ($function == 'transcription') { $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'transcriptions'; + } elseif ($function == 'file') { + $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); + $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'files'; + } elseif ($function == 'assistant') { + $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); + $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'assistans'; + } elseif ($function == 'thread') { + $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); + $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'threads'; + } elseif ($function == 'docparsing') { + $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); + $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'chat/completions'; } else { $this->apiEndpoint = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_URL', $arrayofai[$this->apiService]['url']); $this->apiEndpoint .= (preg_match('/\/$/', $this->apiEndpoint) ? '' : '/').'chat/completions'; } } + if ($moreendpoint) { + $this->apiEndpoint .= '/'.$moreendpoint; + } // $model may be undefined or 'auto'. // If this is the case, we must get it from $function and $this->apiService if (empty($model) || $model == 'auto') { // Return the model from $this->apiService. - if ($function == 'imagegeneration') { + if (in_array($function, array('file', 'assistant', 'thread'))) { + $model = ''; + } elseif ($function == 'imagegeneration') { $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_IMAGE', $arrayofai[$this->apiService][$function]); } elseif ($function == 'audiogeneration') { $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_AUDIO', $arrayofai[$this->apiService][$function]); @@ -132,6 +163,8 @@ class Ai $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TRANSCRIPT', $arrayofai[$this->apiService][$function]); } elseif ($function == 'translation') { $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TRANSLATE', $arrayofai[$this->apiService][$function]); + } elseif ($function == 'docparsing') { + $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_DOCPARSING', $arrayofai[$this->apiService][$function]); } else { // else 'textgenerationemail', 'textgenerationwebpage', 'textgeneration', 'texttranslation', 'textsummarize' $model = getDolGlobalString('AI_API_'.strtoupper($this->apiService).'_MODEL_TEXT', $arrayofai[$this->apiService]['textgeneration']); @@ -182,45 +215,52 @@ class Ai if (empty($prePrompt) && $function == 'textrephraser') { $prePrompt = self::AI_DEFAULT_PROMPT_FOR_TEXT_REPHRASER; } - - $fullInstructions = $instructions.($postPrompt ? (preg_match('/[\.\!\?]$/', $instructions) ? '' : '.').' '.$postPrompt : ''); - - // Set payload string - /*{ - "messages": [ - { - "content": "You are a helpful assistant.", - "role": "system" - }, - { - "content": "Hello!", - "role": "user" - } - ], - "model": "tinyllama-1.1b", - "stream": true, - "max_tokens": 2048, - "stop": [ - "hello" - ], - "frequency_penalty": 0, - "presence_penalty": 0, - "temperature": 0.7, - "top_p": 0.95 - }*/ - - $arrayforpayload = array( - 'messages' => array(array('role' => 'user', 'content' => $fullInstructions)), - 'model' => $model, - ); - - // Add a system message - $addDateTimeContext = false; - if ($addDateTimeContext) { // @phpstan-ignore-line - $prePrompt = ($prePrompt ? $prePrompt.(preg_match('/[\.\!\?]$/', $prePrompt) ? '' : '.').' ' : '').'Today we are '.dol_print_date(dol_now(), 'dayhourtext'); + if (empty($prePrompt) && $function == 'docparsing') { + $prePrompt = self::AI_DEFAULT_PROMPT_FOR_DOC_PARSING; } - if ($prePrompt) { - $arrayforpayload['messages'][] = array('role' => 'system', 'content' => $prePrompt); + + if (is_array($instructions)) { + $arrayforpayload = $instructions; + } else { + $fullInstructions = $instructions.($postPrompt ? (preg_match('/[\.\!\?]$/', $instructions) ? '' : '.').' '.$postPrompt : ''); + + // Set payload string + /*{ + "messages": [ + { + "content": "You are a helpful assistant.", + "role": "system" + }, + { + "content": "Hello!", + "role": "user" + } + ], + "model": "tinyllama-1.1b", + "stream": true, + "max_tokens": 2048, + "stop": [ + "hello" + ], + "frequency_penalty": 0, + "presence_penalty": 0, + "temperature": 0.7, + "top_p": 0.95 + }*/ + + $arrayforpayload = array( + 'messages' => array(array('role' => 'user', 'content' => $fullInstructions)), + 'model' => $model, + ); + + // Add a system message + $addDateTimeContext = false; + if ($addDateTimeContext) { // @phpstan-ignore-line + $prePrompt = ($prePrompt ? $prePrompt.(preg_match('/[\.\!\?]$/', $prePrompt) ? '' : '.').' ' : '').'Today we are '.dol_print_date(dol_now(), 'dayhourtext'); + } + if ($prePrompt) { + $arrayforpayload['messages'][] = array('role' => 'system', 'content' => $prePrompt); + } } /* @@ -229,12 +269,35 @@ class Ai $arrayforpayload['stream'] = false; */ - $payload = json_encode($arrayforpayload); + if ($function == 'docparsing') { + // $arrayforpayload is not used in this case + // $fullInstructions is not used in this case + // $prePrompt is not used in this case + $payload = array( + "name" => "PDF Analyzer", + "model" => $model, + "instructions" => $instructions, + "tools" => [ + ["type" => "file_search"] + ] + ); + } elseif ($function == 'thread') { + $payload = $instructions; + } else { + $payload = json_encode($arrayforpayload); + } $headers = array( 'Authorization: Bearer ' . $this->apiKey, - 'Content-Type: application/json' ); + if ($function != 'file') { + $headers[] = 'Content-Type: application/json'; + } + if (!empty($moreheaders)) { + foreach ($moreheaders as $morekey => $moreval) { + $headers[] = $morekey.': '.$moreval; + } + } if (getDolGlobalString("AI_DEBUG")) { if (@is_writable($dolibarr_main_data_root)) { // Avoid fatal error on fopen with open_basedir @@ -242,8 +305,18 @@ class Ai $fp = fopen($outputfile, "w"); // overwrite if ($fp) { - fwrite($fp, "Call endpoint ".$this->apiEndpoint." with POST and the following HTTP headers and Payload:\n"); + if ($function == 'docparsing') { + fwrite($fp, "Call endpoint ".$this->apiEndpoint." with POST and the following file to upload:\n"); + fwrite($fp, $instructions."\n"); + } else { + fwrite($fp, "Call endpoint ".$this->apiEndpoint." with POST and the following message:\n"); + fwrite($fp, $fullInstructions."\n"); + fwrite($fp, "Prepompt\n"); + fwrite($fp, $prePrompt."\n"); + } + fwrite($fp, "HTTP Header\n"); fwrite($fp, var_export($headers, true)."\n"); + fwrite($fp, "Payload\n"); fwrite($fp, var_export($payload, true)."\n"); fclose($fp); diff --git a/htdocs/ai/lib/ai.lib.php b/htdocs/ai/lib/ai.lib.php index 7fcaa14d185..376515f7035 100644 --- a/htdocs/ai/lib/ai.lib.php +++ b/htdocs/ai/lib/ai.lib.php @@ -50,7 +50,9 @@ function getListOfAIFeatures() 'videogeneration' => array('label' => 'VideoGeneration', 'picto' => '', 'status' => 'notused', 'function' => 'VIDEO'), 'audiogeneration' => array('label' => 'AudioGeneration', 'picto' => '', 'status' => 'notused', 'function' => 'AUDIO'), 'transcription' => array('label' => 'AudioTranscription', 'picto' => '', 'status' => 'notused', 'function' => 'TRANSCRIPT'), - 'translation' => array('label' => 'AudioTranslation', 'picto' => '', 'status' => 'notused', 'function' => 'TRANSLATE') + 'translation' => array('label' => 'AudioTranslation', 'picto' => '', 'status' => 'notused', 'function' => 'TRANSLATE'), + + 'docparsing' => array('label' => 'DocumentPArsing', 'picto' => '', 'status' => 'experimental', 'function' => 'DOCPARSING') ); return $arrayofaifeatures; @@ -77,6 +79,7 @@ function getListOfAIServices() 'videogeneration' => 'na', 'transcription' => 'whisper-1', // audio to text 'translation' => 'whisper-1', // audio to text into another language + 'docparsing' => 'na', ), 'groq' => array( 'label' => 'Groq', @@ -88,6 +91,7 @@ function getListOfAIServices() 'videogeneration' => 'na', 'transcription' => 'na', 'translation' => 'na', + 'docparsing' => 'na', ), 'mistral' => array( 'label' => 'Mistral', @@ -99,6 +103,7 @@ function getListOfAIServices() 'videogeneration' => 'na', 'transcription' => 'na', 'translation' => 'na', + 'docparsing' => 'open-mistral-7b', ), 'custom' => array( 'label' => 'Custom', @@ -110,6 +115,7 @@ function getListOfAIServices() 'videogeneration' => 'na', 'transcription' => 'mixtral-8x7b-32768', 'translation' => 'mixtral-8x7b-32768', + 'docparsing' => 'na', ) //'gemini' => array( // 'label' => 'Gemini', diff --git a/htdocs/core/actions_linkedfiles.inc.php b/htdocs/core/actions_linkedfiles.inc.php index e59fc651416..546d3835c2a 100644 --- a/htdocs/core/actions_linkedfiles.inc.php +++ b/htdocs/core/actions_linkedfiles.inc.php @@ -29,6 +29,7 @@ * @var User $user * * @var string $action + * @var string $backtopage * @var string $upload_dir * @var string $upload_dirold * @var string $confirm @@ -109,6 +110,7 @@ if (GETPOST('sendit', 'alpha') && getDolGlobalString('MAIN_UPLOAD_DOC') && !empt if (!empty($upload_dirold) && getDolGlobalInt('PRODUCT_USE_OLD_PATH_FOR_PHOTO')) { $result = dol_add_file_process($upload_dirold, $allowoverwrite, 1, 'userfile', GETPOST('savingdocmask', 'alpha'), null, '', $generatethumbs, $object, empty($forceFullTextIndexation) ? 0 : $forceFullTextIndexation); } elseif (!empty($upload_dir)) { + var_dump($upload_dir); $result = dol_add_file_process($upload_dir, $allowoverwrite, 1, 'userfile', GETPOST('savingdocmask', 'alpha'), null, '', $generatethumbs, $object, empty($forceFullTextIndexation) ? 0 : $forceFullTextIndexation); } } diff --git a/htdocs/core/ajax/ajaxstatusprospect.php b/htdocs/core/ajax/ajaxstatusprospect.php index a7ac677db3f..680b43ae30d 100644 --- a/htdocs/core/ajax/ajaxstatusprospect.php +++ b/htdocs/core/ajax/ajaxstatusprospect.php @@ -44,8 +44,6 @@ if (!defined('NOREQUIRESOC')) { // Load Dolibarr environment require '../../main.inc.php'; -require_once DOL_DOCUMENT_ROOT.'/societe/class/client.class.php'; - /** * @var Conf $conf * @var DoliDB $db @@ -53,6 +51,7 @@ require_once DOL_DOCUMENT_ROOT.'/societe/class/client.class.php'; * @var Translate $langs * @var User $user */ +require_once DOL_DOCUMENT_ROOT.'/societe/class/client.class.php'; $idstatus = GETPOSTINT('id'); $idprospect = GETPOSTINT('prospectid'); diff --git a/htdocs/core/ajax/ajaxuploadpage.php b/htdocs/core/ajax/ajaxuploadpage.php new file mode 100644 index 00000000000..20e6a46bd44 --- /dev/null +++ b/htdocs/core/ajax/ajaxuploadpage.php @@ -0,0 +1,444 @@ + + * Copyright (C) 2024-2025 Frédéric France + * Copyright (C) 2024 MDW + * + * This file is a modified version of datepicker.php from phpBSM to fix some + * bugs, to add new features and to dramatically increase speed. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** + * \file htdocs/core/ajax/ajaxuploadpage.php + * \brief Page to show a generic upload file feature + */ + +require_once '../main.inc.php'; +/** + * @var Conf $conf + * @var DoliDB $db + * @var HookManager $hookmanager + * @var Translate $langs + * @var User $user + */ +require_once DOL_DOCUMENT_ROOT.'/core/class/html.form.class.php'; +require_once DOL_DOCUMENT_ROOT.'/core/lib/files.lib.php'; +require_once DOL_DOCUMENT_ROOT.'/ai/class/ai.class.php'; + + +if (GETPOST('lang', 'aZ09')) { + $langs->setDefaultLang(GETPOST('lang', 'aZ09')); // If language was forced on URL by the main.inc.php +} + +$langs->loadLangs(array("main", "other", "exports")); + +$action = GETPOST('action', 'aZ09'); +$modulepart = GETPOST('modulepart', 'aZ09'); + +$upload_dir = $conf->users->dir_temp.'/import'; + +// Delete the temporary files that are used when uploading files +//dol_delete_file($upload_dir.'/upload_page-by'.$user->id.'-*'); + +$file = GETPOST('file'); + +$reg = array(); +if (preg_match('/^upload_page-([a-z_]+)-uid(\d+)-/', $file, $reg)) { + $modulepart = $reg[1]; + + if ($reg[2] != $user->id) { + accessforbidden('User id found in filename to process does not match current user id'); + } +} else { + accessforbidden('Bad value for file parameter'); +} + +$error = 0; +$errors = array(); + + +/* + * Actions + */ + +// + + + +/* + * View + */ + +top_httphead('application/json'); + +$originalfilename = $file; + +$uid = $thiid = $pid = $erid = $salid = 0; +if (preg_match('/-uid([\d+])/', $file, $reg)) { + $uid = $reg[1]; + $originalfilename = preg_replace('/-uid\d+/', '', $originalfilename); +} +if (preg_match('/-thiid([\d+])/', $file, $reg)) { + $thiid = $reg[1]; + $originalfilename = preg_replace('/-thiid\d+/', '', $originalfilename); +} +if (preg_match('/-pid([\d+])/', $file, $reg)) { + $pid = $reg[1]; + $originalfilename = preg_replace('/-pid\d+/', '', $originalfilename); +} +if (preg_match('/-erid([\d+])/', $file, $reg)) { + $erid = $reg[1]; + $originalfilename = preg_replace('/-erid\d+/', '', $originalfilename); +} +if (preg_match('/-salid([\d+])/', $file, $reg)) { + $salid = $reg[1]; + $originalfilename = preg_replace('/-salid\d+/', '', $originalfilename); +} + +$originalfilename = preg_replace('/^upload_page-[a-z_]+-/', '', $originalfilename); + + +$ai = new Ai($db); + + +print $langs->trans("ImportInProcess", $originalfilename).'
'; +print '
'; + +print $langs->trans("AIProcessingPleaseWait", $ai->getApiService()).'...'; +print '
'; + +print '
+
+
'; + + +print ''; +print "\n\n"; + + + + +//$METHOD = 'thread'; // For ChatGPT. +$METHOD = 'converttotext'; // For Mistral and most others + + +$docformat = $doctypelabel = $prompt = ''; +$fullpathoffile = $upload_dir.'/'.$file; +$answer = null; + +// Set the prompting +if (!$error && $modulepart == 'invoice_supplier') { + $docformat = 'pdf'; + $doctypelabel = 'invoice'; + + // Call IA to get information on PDF + $prompt = 'Analyze the contents of this '.$docformat.' document of an '.$doctypelabel.' and convert all readable text and numerical information into a structured JSON format. Follow these guidelines: + + 1. **Hierarchical Structure:** Group related information into logical categories like "document_info", "vendor or issuer", "recipient", "items", "payment_info" and "other" depending on the content. + 2. **Flexible Field Identification:** Identify and organize common fields that are found such as: + - "invoice number", "invoice reference" + - "date", "issue date", "due date", "transaction or invoice date" + - "vendor name", "vendor vat number", "vendor professional id (siret, siren, ...), "vendor address", "vendor phone", "vendor email", + - "items", "products", "services", "product or service label", "product or service ref", "product or service ref" + - "totals", "summary", "amounts", "balance" + - "notes," "comments," "messages," "terms" + 3. **Handle Tables and Lists:** If the document contains tables, represent each row as an object in a list with fields like "no", "ref", "description", "quantity", "vat rate", "unit price", "total excluding tax", "total including tax". + 4. **Normalize Dates:** If dates are present, format them in ISO format (YYYY-MM-DD) whenever possible. + 5. **Ignore Background Noise:** Exclude background noise, decorative elements, and irrelevant symbols that do not contribute to the data content. + 6. **Preserve Context:** If the image contains sections, headings, or grouping indicators, use them to create logical hierarchies in the JSON structure. + 7. **General Usability:** Format the text to be suitable for further processing, analysis, or database import. + + **Example JSON Structure:** + + ```json + { + "document_info": { + "document_ref": "", + "date": "", + "title": "" + }, + "vendor": { + "name": "<name>", + "address": "<address>", + "phone": "<phone number>", + "email": "<email>" + "vatnumber": "<vat number>", + "prodif": "<professional id>", + }, + "recipient": { + "name": "<name>", + "address": "<address>", + "phone": "<phone number>", + "email": "<email>" + "vatnumber": "<vat number>", + "prodif": "<professional id>", + }, + "items": [ + { + "no": "<number>", + "ref": "<ref>", + "label": "<label>", + "description": "<description>", + "quantity": "<quantity>", + "vatrate": "<vat rate>", + "totalinctax": "<total including tax>" + "totalexcltax": "<total excluding tax>" + } + ], + "summary": { + "subtotal": "<subtotal>", + "tax": "<amount of tax>", + "total": "<total to pay>" + }, + "payment_methods": [ + { + "method": "<check or cash or card or direct_debit or credit_transferor other>", + "details": "Detail of the payment mode" + } + ], + "payments_done": [ + { + "method": "<check or cash or card or direct_debit or credit_transferor other>", + "amount": "<detail of the payment>", + "note":"<other information on payment done> + } + ], + "notes": "<optional text>" + } + '; +} + + +// TODO Move this into an AJAX service and just output the JS code to call the aajax to start + +if ($METHOD == 'converttotext') { + $fileContent = 'eee'; + + $prompt = 'This is the content of the document:'."\n\n"; + + $prompt .= $fileContent; + + $result = $ai->generateContent($prompt, 'auto', 'docparsing', ''); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $answer = $result; + } +} + + +if ($METHOD == 'thread') { + $prompt = ''; + + $fileId = 0; + $assistantId = 0; + $threadId = 0; + $runId = 0; + + + // First part is to send the file + $payload = array( + "purpose" => "assistants", + "file" => new CURLFile($fullpathoffile) + ); + + $result = $ai->generateContent($payload, 'auto', 'file', ''); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $fileId = json_decode($result, true)['id']; + } + + + // Create assistant + if (!$error) { + $payload = 'Analyze PDF and answer precisely'; + + $result = $ai->generateContent($payload, 'auto', 'assistant', '', array('OpenAI-Beta', 'assistants=v2')); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $assistantId = json_decode($result, true)['id']; + } + } + + + // Create thread + if (!$error) { + $payload = '{}'; + + $result = $ai->generateContent($payload, 'auto', 'thread', '', array('OpenAI-Beta', 'assistants=v2')); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $threadId = json_decode($result, true)['id']; + } + } + + + // Add file to thread + if (!$error) { + $payload = array( + "role" => "user", + "content" => [ + [ + "type" => "input_text", + "text" => $prompt + ] + ], + "attachments" => [ + [ + "file_id" => $fileId, + "tools" => [["type" => "file_search"]] + ] + ] + ); + $moreendpoint = $threadId.'/messages'; + + $result = $ai->generateContent($payload, 'auto', 'thread', '', array('OpenAI-Beta', 'assistants=v2'), $moreendpoint); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } + } + + + // Run the thread + if (!$error) { + $payload = array( + ["assistant_id" => $assistantId] + ); + $moreendpoint = $threadId.'/runs'; + + $result = $ai->generateContent($payload, 'auto', 'thread', '', array('OpenAI-Beta', 'assistants=v2'), $moreendpoint); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $runId = json_decode($result, true)['id']; + } + } + + + // Poll until end of thread + if (!$error) { + do { + sleep(1); + + $payload = ''; + $moreendpoint = $threadId.'/runs/'.$runId; + + $result = $ai->generateContent($payload, 'auto', 'thread', '', array('OpenAI-Beta', 'assistants=v2'), $moreendpoint); + + $threadId = 0; + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + + $status = 'completed'; + } else { + $status = json_decode($result, true)['status']; + } + } while ($status !== "completed"); + } + + + // Get answer + if (!$error) { + $payload = ''; + $moreendpoint = $threadId.'/messages'; + + $result = $ai->generateContent($prompt, 'auto', 'thread', '', array('OpenAI-Beta', 'assistants=v2')); + + if (!empty($result['error']) || !empty($result['curl_error_no']) || (!empty($result['http_code']) && (int) $result['http_code'] !=200)) { + if ($result['error']) { + $error++; + $errors[] = $result['error']; + } + if ($result['curl_error_no']) { + $error++; + $errors[] = $result['curl_error_no']; + } + } else { + $answer = $result; + } + } +} + + +if (!empty($errors)) { + dol_htmloutput_errors('', $errors); +} else { + // Add JS code to scan regularly the status of work and show output and redirect + + var_dump($answer); +} + + +// End of page +llxFooter(); +$db->close(); diff --git a/htdocs/core/upload_page.php b/htdocs/core/upload_page.php index a866a54006b..a7c8ccc355a 100644 --- a/htdocs/core/upload_page.php +++ b/htdocs/core/upload_page.php @@ -46,7 +46,9 @@ $langs->loadLangs(array("main", "other")); $action = GETPOST('action', 'aZ09'); $modulepart = GETPOST('modulepart', 'aZ09'); -$upload_dir = $conf->admin->dir_temp.'/import'; +// users/temp/import +$upload_dir = $conf->user->dir_temp.'/import'; +dol_mkdir($upload_dir); // Delete the temporary files that are used when uploading files dol_delete_file($upload_dir.'/upload_page-by'.$user->id.'-*'); @@ -71,6 +73,7 @@ if ($action == 'uploadfile') { // Test on permission not required here. Done lat exit(1); } + // $modulepart can be 'invoice_supplier', ... $arrayobject = getElementProperties($modulepart); $module = $arrayobject['module']; @@ -84,11 +87,11 @@ if ($action == 'uploadfile') { // Test on permission not required here. Done lat if (in_array($modulepart, array('fournisseur', 'invoice_supplier'))) { $permlevel1 = 'facture'; $permlevel2 = 'read'; - $fileprefix = 'upload_page-by'.$user->id.'-'.$modulepart.'-'.(GETPOSTINT('socid') > 0 ? GETPOSTINT('socid') : 0).'-'.(GETPOSTINT('search_prodid') > 0 ? GETPOSTINT('search_prodid') : 0); + $fileprefix = 'upload_page-'.$modulepart.'-uid'.$user->id.'-thiid'.(GETPOSTINT('socid') > 0 ? GETPOSTINT('socid') : 0).'-pid'.(GETPOSTINT('search_prodid') > 0 ? GETPOSTINT('search_prodid') : 0); } elseif ($modulepart == 'expensereport') { - $fileprefix = 'upload_page-by'.$user->id.'-'.$modulepart.'-'.(GETPOSTINT('userexpensereportid') > 0 ? GETPOSTINT('userexpensereportid') : 0).'-'.(GETPOSTINT('search_prodid') > 0 ? GETPOSTINT('search_prodid') : 0); + $fileprefix = 'upload_page-'.$modulepart.'-uid'.$user->id.'-erid'.(GETPOSTINT('userexpensereportid') > 0 ? GETPOSTINT('userexpensereportid') : 0).'-pid'.(GETPOSTINT('search_prodid') > 0 ? GETPOSTINT('search_prodid') : 0); } elseif ($modulepart == 'salaries') { - $fileprefix = 'upload_page-by'.$user->id.'-'.$modulepart.'-'.(GETPOSTINT('usersalaryid') > 0 ? GETPOSTINT('usersalaryid') : 0); + $fileprefix = 'upload_page-'.$modulepart.'-uid'.$user->id.'-salid'.(GETPOSTINT('usersalaryid') > 0 ? GETPOSTINT('usersalaryid') : 0); } if ($permlevel2) { @@ -100,13 +103,18 @@ if ($action == 'uploadfile') { // Test on permission not required here. Done lat if (!empty($_FILES['userfile']['name'])) { - $_FILES['userfile']['name'] = $fileprefix.'-'.$_FILES['userfile']['name']; + $fullnewname = $fileprefix.'-'.$_FILES['userfile']['name']; + $_FILES['userfile']['name'] = $fullnewname; + // $dir_output = output dir of object + // $dir_temp = temp dir of object + // $upload_dir is "users/temp/import" include DOL_DOCUMENT_ROOT.'/core/actions_linkedfiles.inc.php'; + // TODO Add a js call of ajax service and show instead a message // @phpstan-ignore-next-line $error may have been modified by actions_linkedfiles.inc.php if (!$error) { - header("Location: ".DOL_URL_ROOT.'/core/upload_page2.php?file='.urlencode($fileprefix)); + header("Location: ".DOL_URL_ROOT.'/core/ajax/ajax_upload_page.php?file='.urlencode($fullnewname)); exit; } } diff --git a/htdocs/core/upload_page2.php b/htdocs/core/upload_page2.php deleted file mode 100644 index 3985f4f5721..00000000000 --- a/htdocs/core/upload_page2.php +++ /dev/null @@ -1,119 +0,0 @@ -<?php -/* Copyright (C) 2005-2017 Laurent Destailleur <eldy@users.sourceforge.net> - * Copyright (C) 2024-2025 Frédéric France <frederic.france@free.fr> - * Copyright (C) 2024 MDW <mdeweerd@users.noreply.github.com> - * - * This file is a modified version of datepicker.php from phpBSM to fix some - * bugs, to add new features and to dramatically increase speed. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see <https://www.gnu.org/licenses/>. - */ - -/** - * \file htdocs/core/upload_page2.php - * \brief Page to show a generic upload file feature - */ - -require_once '../main.inc.php'; -/** - * @var Conf $conf - * @var DoliDB $db - * @var HookManager $hookmanager - * @var Translate $langs - * @var User $user - */ -require_once DOL_DOCUMENT_ROOT.'/core/class/html.form.class.php'; -require_once DOL_DOCUMENT_ROOT.'/core/lib/files.lib.php'; - - -if (GETPOST('lang', 'aZ09')) { - $langs->setDefaultLang(GETPOST('lang', 'aZ09')); // If language was forced on URL by the main.inc.php -} - -$langs->loadLangs(array("main", "other")); - -$action = GETPOST('action', 'aZ09'); -$modulepart = GETPOST('modulepart', 'aZ09'); - -$upload_dir = $conf->admin->dir_temp.'/import'; - -// Delete the temporary files that are used when uploading files -//dol_delete_file($upload_dir.'/upload_page-by'.$user->id.'-*'); - -$file = GETPOST('file'); - -$reg = array(); -if (preg_match('/^upload_page-by(\d+)-([a-z_]+)-(\d+)/', $file, $reg)) { - $modulepart = $reg[2]; - - if ($reg[1] != $user->id) { - accessforbidden('User id for file to process does not match current user id'); - } -} else { - accessforbidden('Bad value for file parameter'); -} - - -/* - * Actions - */ - -// - - - -/* - * View - */ - -//$form = new Form($db); - -$title = $langs->trans("UploadFile"); -$help_url = ''; - -$arrayofjs = array(); -$arrayofcss = array(); - -llxHeader('', $title, $help_url, '', 0, 0, $arrayofjs, $arrayofcss, '', 'mod-upload page-card'); -//top_htmlhead($head, $title, 0, 0, $arrayofjs, $arrayofcss); - -print load_fiche_titre('', '', '', 0, '', '', '<h2>'.img_picto('', 'upload').' '.$title.'</h2>'); - -// Show all forms -print "\n"; -print "<!-- Begin Form -->\n"; -print '<form id="uploadform" enctype="multipart/form-data" method="POST" action="'.dolBuildUrl($_SERVER["PHP_SELF"]).'">'; -print '<input type="hidden" name="token" value="'.newToken().'">'; -print '<input type="hidden" name="action" value="uploadfile">'; -print '<input type="hidden" name="sendit" value="1">'; -print '<input type="hidden" name="modulepart" id="modulepart" value="">'; -print '<input type="hidden" name="overwritefile" value="1">'; - -print "<script> -$(document).ready(function() { - -}); -</script>"; - -print $file; - - -print '</form>'; -print "\n<!-- End Form -->\n"; - - - -// End of page -llxFooter(); -$db->close(); diff --git a/htdocs/langs/en_US/admin.lang b/htdocs/langs/en_US/admin.lang index aff29a56063..34b3a1887c2 100644 --- a/htdocs/langs/en_US/admin.lang +++ b/htdocs/langs/en_US/admin.lang @@ -2583,6 +2583,7 @@ AI_API_MODEL_AUDIO=AI model for audio feature AI_API_MODEL_TRANSCRIPT=AI model for transcription feature AI_API_MODEL_TRANSLATE=AI model for translation feature AI_API_MODEL_VIDEO=AI model for video feature +AI_API_MODEL_DOCPARSING=AI model for document parsing AiSetup=AI module setup AiCustomPrompt=AI custom prompt AI_CONFIGURATIONS_PROMPT=Custom prompt @@ -2591,8 +2592,8 @@ TextTranslation=Text translation ImageGeneration=Image generation VideoGeneration=Video generation AudioGeneration=Audio generation -AIPromptForFeatures=AI custom prompts (AI service %s) -AIModelForFeature=AI model per features (AI service %s) +AIPromptForFeatures=AI custom prompts (AI provider: %s) +AIModelForFeature=AI model per features (AI provider: %s) NewCustomPrompt=New custom prompt AudioTranscription=Audio transcription AudioTranslation=Audio translation diff --git a/htdocs/langs/en_US/exports.lang b/htdocs/langs/en_US/exports.lang index 521e6dc136a..1c18e9f8b91 100644 --- a/htdocs/langs/en_US/exports.lang +++ b/htdocs/langs/en_US/exports.lang @@ -153,3 +153,4 @@ AllTargetMandatoryFieldsAreMapped=All target fields that need a mandatory value ResultOfSimulationNoError=Result of simulation: No error NumberOfLinesLimited=Number of lines limited ErrorFileLinesReachEOF=File has %s lines. However we reach the end of file or an empty line at record %s. This may occurs when some records are split onto several lines and not correctly delimited by the "Char delimiter", or if there is line with no data on all fields. +ImportInProcess=Import of file %s is in process