From 7e60db00b7bb66658a21662f2c46b9544fa4fb49 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 27 Aug 2024 21:50:44 +0200 Subject: [PATCH 1/4] Fix log of emailcollector --- .../emailcollector/class/emailcollector.class.php | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/htdocs/emailcollector/class/emailcollector.class.php b/htdocs/emailcollector/class/emailcollector.class.php index e28e943f0a4..31209e2692b 100644 --- a/htdocs/emailcollector/class/emailcollector.class.php +++ b/htdocs/emailcollector/class/emailcollector.class.php @@ -1681,7 +1681,6 @@ class EmailCollector extends CommonObject $matches = array(); preg_match_all('/([^: ]+): (.+?(?:\r\n\s(?:.+?))*)\r\n/m', $header, $matches); $headers = array_combine($matches[1], $matches[2]); - //var_dump($headers);exit; if (!empty($headers['in-reply-to']) && empty($headers['In-Reply-To'])) { $headers['In-Reply-To'] = $headers['in-reply-to']; @@ -1698,7 +1697,13 @@ class EmailCollector extends CommonObject $headers['Subject'] = $this->decodeSMTPSubject($headers['Subject']); - $emailto = $this->decodeSMTPSubject($overview[0]->to); + if (getDolGlobalInt('MAIN_IMAP_USE_PHPIMAP')) { + $emailto = (string) $overview['to']; + } else { + $emailto = $this->decodeSMTPSubject($overview[0]->to); + } + //var_dump($headers); + //var_dump($overview);exit; $operationslog .= '
** Process email #'.dol_escape_htmltag($iforemailloop); @@ -1710,9 +1715,11 @@ class EmailCollector extends CommonObject $operationslog .= " - ".dol_escape_htmltag((string) $imapemail); $msgid = str_replace(array('<', '>'), '', $overview[0]->message_id); } - $operationslog .= " - MsgId: ".$msgid." - References: ".dol_escape_htmltag($headers['References'] ?? '')." - Subject: ".dol_escape_htmltag($headers['Subject']); + $operationslog .= " - MsgId: ".$msgid; + $operationslog .= " - Date: ".($headers['Date'] ?? ''); + $operationslog .= " - References: ".dol_escape_htmltag($headers['References'] ?? '')." - Subject: ".dol_escape_htmltag($headers['Subject']); - dol_syslog("-- Process email ".$iforemailloop." References: ".($headers['References'] ?? '')." Subject: ".$headers['Subject']); + dol_syslog("-- Process email #".$iforemailloop." MsgId: ".$msgid." Date: ".($headers['Date'] ?? '')." References: ".($headers['References'] ?? '')." Subject: ".$headers['Subject']); $trackidfoundintorecipienttype = ''; From 367cdd7ccc3d92dda569a2d89cb6f55826fef897 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 27 Aug 2024 22:20:51 +0200 Subject: [PATCH 2/4] Fix emailcollector from tickets --- .../emailcollector/class/emailcollector.class.php | 4 ++-- htdocs/langs/en_US/mails.lang | 2 +- htdocs/ticket/card.php | 14 ++++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/htdocs/emailcollector/class/emailcollector.class.php b/htdocs/emailcollector/class/emailcollector.class.php index 31209e2692b..87fa039725e 100644 --- a/htdocs/emailcollector/class/emailcollector.class.php +++ b/htdocs/emailcollector/class/emailcollector.class.php @@ -1406,14 +1406,14 @@ class EmailCollector extends CommonObject } } - if (empty($targetdir)) { // Use last date as filter if there is no targetdir defined. + if (empty($targetdir) || !getDolGlobalString('EMAILCOLLECTOR_NO_FILTER_ON_DATE_IF_THERE_IS_A_TARGETDIR')) { // Use last date as filter if there is no targetdir defined. $fromdate = 0; if ($this->datelastok) { $fromdate = $this->datelastok; } if ($fromdate > 0) { // $search .= ($search ? ' ' : '').'SINCE '.date('j-M-Y', $fromdate - 1); // SENTSINCE not supported. Date must be X-Abc-9999 (X on 1 digit if < 10) - array_push($criteria, array("SINCE" => date('j-M-Y', $fromdate - 1))); + array_push($criteria, array("SINCE" => date('j-M-Y', $fromdate - 1))); // -1 is to add a security to no forgot some email } //$search.=($search?' ':'').'SINCE 8-Apr-2022'; } diff --git a/htdocs/langs/en_US/mails.lang b/htdocs/langs/en_US/mails.lang index 3570e6a36c4..55fcf6d0bfe 100644 --- a/htdocs/langs/en_US/mails.lang +++ b/htdocs/langs/en_US/mails.lang @@ -190,7 +190,7 @@ NoMoreRecipientToSendTo=No more recipient to send the email to EmailOptedOut=Email owner has requested to not contact him with this email anymore EvenUnsubscribe=Include opt-out emails EvenUnsubscribeDesc=Include opt-out emails when you select emails as targets. Useful for mandatory service emails for example. -XEmailsDoneYActionsDone=%s emails pre-qualified, %s emails successfully processed (for %s record/actions done) +XEmailsDoneYActionsDone=%s emails pre-qualified, %s emails successfully processed (for %s operations done) YouCanMakeSomeInstructionForEmail=You can make some instructions for your Email (Example: generate image in email template...) ModelTemplate=Email template YouCanChooseAModelForYouMailContent= You can choose one of template models or generate one with AI diff --git a/htdocs/ticket/card.php b/htdocs/ticket/card.php index 89ddb7179c7..a6121973fe0 100644 --- a/htdocs/ticket/card.php +++ b/htdocs/ticket/card.php @@ -897,15 +897,25 @@ if ($action == 'create' || $action == 'presend') { $createdbyshown++; } + $createdfrompublicticket = 0; + $createdfromemailcollector = 0; + if (!empty($object->origin_email) && (empty($object->email_msgid) || preg_match('/dolibarr\-tic\d+/', $object->email_msgid))) { + // If ticket create from public interface - TODO Add a more robust test to know if created by public interface + $createdfrompublicticket = 1; + } elseif (!empty($object->email_msgid)) { + // If ticket create by emailcollector - TODO Add a more robust test to know if created by email collector (using import ky ?) + $createdfromemailcollector = 1; + } + //var_dump($object); - if (!empty($object->origin_email)) { // If ticket create from public interface - TODO Add a more robust test to know if created by pubic interface + if ($createdfrompublicticket) { $htmltooptip = $langs->trans("OriginEmail").': '.$object->origin_email; $htmltooptip .= '
'.$langs->trans("IP").': '.$object->ip; $morehtmlref .= ($createdbyshown ? ' - ' : '
'); $morehtmlref .= ($createdbyshown ? '' : $langs->trans("CreatedBy").' : '); $morehtmlref .= img_picto('', 'email', 'class="paddingrightonly"'); $morehtmlref .= dol_escape_htmltag($object->origin_email).' - '.$form->textwithpicto($langs->trans("CreatedByPublicPortal"), $htmltooptip, 1, 'help', '', 0, 3, 'tooltip').''; - } elseif (!empty($object->email_msgid)) { // If ticket create by emailcollector - TODO Add a more robust test to know if created by email collector (using import ky ?) + } elseif ($createdfromemailcollector) { $langs->load("mails"); $htmltooltip = $langs->trans("EmailMsgID").': '.$object->email_msgid; $htmltooltip .= '
'.$langs->trans("EmailDate").': '.dol_print_date($object->email_date, 'dayhour'); From 7464e22a4d17d168df2d0f3f311e7aa7e988f6de Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Wed, 28 Aug 2024 00:19:23 +0200 Subject: [PATCH 3/4] Debug emailcollector with PHP-IMAP lib --- .../codespell/codespell-lines-ignore.txt | 1 + .../class/emailcollector.class.php | 148 ++++++++++++------ 2 files changed, 98 insertions(+), 51 deletions(-) diff --git a/dev/tools/codespell/codespell-lines-ignore.txt b/dev/tools/codespell/codespell-lines-ignore.txt index 4d4b5db599f..6e8868933f3 100644 --- a/dev/tools/codespell/codespell-lines-ignore.txt +++ b/dev/tools/codespell/codespell-lines-ignore.txt @@ -19,6 +19,7 @@ print ''; $date = $overview[0]->udate; $dateb = $this->db->jdate($data[$j]->datea); + $dateemail = dol_stringtotime((string) $overview[0]->udate, 'gmt'); $link->datea = $this->db->jdate($obj->datea); $object->periode = $newdateperiod; $out .= dol_print_date($file->datea, 'dayhour'); diff --git a/htdocs/emailcollector/class/emailcollector.class.php b/htdocs/emailcollector/class/emailcollector.class.php index 87fa039725e..342bb9cd8cf 100644 --- a/htdocs/emailcollector/class/emailcollector.class.php +++ b/htdocs/emailcollector/class/emailcollector.class.php @@ -111,13 +111,13 @@ class EmailCollector extends CommonObject // BEGIN MODULEBUILDER PROPERTIES /** - * @var array|string,position:int,notnull?:int,visible:int,noteditable?:int,default?:string,index?:int,foreignkey?:string,searchall?:int,isameasure?:int,css?:string,csslist?:string,help?:string,showoncombobox?:int,disabled?:int,arrayofkeyval?:array,comment?:string}> Array with all fields and their property. Do not use it as a static var. It may be modified by constructor. + * @var array|string,position:int,notnull?:int,visible:int<-2,5>|string,noteditable?:int<0,1>,default?:string,index?:int,foreignkey?:string,searchall?:int<0,1>,isameasure?:int<0,1>,css?:string,csslist?:string,help?:string,showoncombobox?:int<0,2>,disabled?:int<0,1>,arrayofkeyval?:array,comment?:string,validate?:int<0,1>}> Array with all fields and their property. Do not use it as a static var. It may be modified by constructor. */ public $fields = array( 'rowid' => array('type' => 'integer', 'label' => 'TechnicalID', 'visible' => 2, 'enabled' => 1, 'position' => 1, 'notnull' => 1, 'index' => 1), 'entity' => array('type' => 'integer', 'label' => 'Entity', 'enabled' => 1, 'visible' => 0, 'default' => 1, 'notnull' => 1, 'index' => 1, 'position' => 20), 'ref' => array('type' => 'varchar(128)', 'label' => 'Ref', 'enabled' => 1, 'visible' => 1, 'notnull' => 1, 'showoncombobox' => 1, 'index' => 1, 'position' => 10, 'searchall' => 1, 'help' => 'Example: MyCollector1', 'csslist' => 'tdoverflowmax200'), - 'label' => array('type' => 'varchar(255)', 'label' => 'Label', 'visible' => 1, 'enabled' => 1, 'position' => 30, 'notnull' => -1, 'searchall' => 1, 'help' => 'Example: My Email collector', 'csslist' => 'tdoverflowmax150', 'tdcss'=>'titlefieldmiddle'), + 'label' => array('type' => 'varchar(255)', 'label' => 'Label', 'visible' => 1, 'enabled' => 1, 'position' => 30, 'notnull' => -1, 'searchall' => 1, 'help' => 'Example: My Email collector', 'csslist' => 'tdoverflowmax150', 'tdcss' => 'titlefieldmiddle'), 'description' => array('type' => 'text', 'label' => 'Description', 'visible' => -1, 'enabled' => 1, 'position' => 60, 'notnull' => -1, 'cssview' => 'small', 'csslist' => 'small tdoverflowmax200'), 'host' => array('type' => 'varchar(255)', 'label' => 'EMailHost', 'visible' => 1, 'enabled' => 1, 'position' => 90, 'notnull' => 1, 'searchall' => 1, 'comment' => "IMAP server", 'help' => 'Example: imap.gmail.com', 'csslist' => 'tdoverflowmax125'), 'port' => array('type' => 'varchar(10)', 'label' => 'EMailHostPort', 'visible' => 1, 'enabled' => 1, 'position' => 91, 'notnull' => 1, 'searchall' => 0, 'comment' => "IMAP server port", 'help' => 'Example: 993', 'csslist' => 'tdoverflowmax50', 'default' => '993'), @@ -178,11 +178,6 @@ class EmailCollector extends CommonObject */ public $status; - /** - * @var integer|string date_creation - */ - public $date_creation; - /** * @var int ID */ @@ -212,7 +207,7 @@ class EmailCollector extends CommonObject public $maxemailpercollect; /** - * @var integer|string $datelastresult + * @var int|string $datelastresult */ public $datelastresult; @@ -461,7 +456,7 @@ class EmailCollector extends CommonObject * @param string $sortorder sorting order * @param int $limit sort limit * @param int $page page to start on - * @return array Array with key => EmailCollector object + * @return EmailCollector[] Array with key => EmailCollector object */ public function fetchAll(User $user, $activeOnly = 0, $sortfield = 's.rowid', $sortorder = 'ASC', $limit = 100, $page = 0) { @@ -1178,16 +1173,17 @@ class EmailCollector extends CommonObject // } // Token expired so we refresh it if (is_object($tokenobj) && $expire) { - $this->debuginfo .= 'Refresh token
'; + $this->debuginfo .= 'Refresh token '.$OAUTH_SERVICENAME.'
'; $credentials = new Credentials( getDolGlobalString('OAUTH_'.$this->oauth_service.'_ID'), getDolGlobalString('OAUTH_'.$this->oauth_service.'_SECRET'), - getDolGlobalString('OAUTH_'.$this->oauth_service.'_URLAUTHORIZE') + getDolGlobalString('OAUTH_'.$this->oauth_service.'_URLCALLBACK') ); $serviceFactory = new \OAuth\ServiceFactory(); $oauthname = explode('-', $OAUTH_SERVICENAME); // ex service is Google-Emails we need only the first part Google $apiService = $serviceFactory->createService($oauthname[0], $credentials, $storage, array()); + '@phan-var-force OAuth\OAuth2\Service\AbstractService|OAuth\OAuth1\Service\AbstractService $apiService'; // createService is only ServiceInterface // We have to save the token because Google give it only once $refreshtoken = $tokenobj->getRefreshToken(); $tokenobj = $apiService->refreshAccessToken($tokenobj); @@ -1582,7 +1578,7 @@ class EmailCollector extends CommonObject if ($f) { $folder = $f[0]; if ($folder instanceof Webklex\PHPIMAP\Folder) { - $Query = $folder->messages()->where($criteria); + $Query = $folder->messages()->where($criteria); // @phan-suppress-current-line PhanPluginUnknownObjectMethodCall } else { $error++; $this->error = "Source directory ".$sourcedir." not found"; @@ -1609,6 +1605,7 @@ class EmailCollector extends CommonObject return -1; } + '@phan-var-force Webklex\PHPIMAP\Query\Query $Query'; try { //var_dump($Query->count()); if ($mode > 0) { @@ -1669,7 +1666,8 @@ class EmailCollector extends CommonObject // GET header and overview datas if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { - $header = $imapemail->getHeader()->raw; + '@phan-var-force Webklex\PHPIMAP\Message $imapemail'; + $header = $imapemail->getHeader()->raw; // @phan-suppress-current-line PhanPluginUnknownObjectMethodCall // @phan-suppress-current-line PhanPluginUnknownObjectMethodCall $overview = $imapemail->getAttributes(); } else { $header = imap_fetchheader($connection, $imapemail, FT_UID); @@ -1709,6 +1707,7 @@ class EmailCollector extends CommonObject if (getDolGlobalInt('MAIN_IMAP_USE_PHPIMAP')) { /** @var Webklex\PHPIMAP\Message $imapemail */ + '@phan-var-force Webklex\PHPIMAP\Message $imapemail'; // $operationslog .= " - ".dol_escape_htmltag((string) $imapemail); $msgid = str_replace(array('<', '>'), '', $overview['message_id']); } else { @@ -1816,7 +1815,7 @@ class EmailCollector extends CommonObject $isreplytook = 0; foreach ($rulesreplyto as $key => $rulereplyto) { if (preg_match('/'.preg_quote($rulereplyto, '/').'/', $headers['Reply-To'])) { - $isreplytook ++; + $isreplytook++; } } @@ -1864,6 +1863,7 @@ class EmailCollector extends CommonObject if (getDolGlobalInt('MAIN_IMAP_USE_PHPIMAP')) { /** @var Webklex\PHPIMAP\Message $imapemail */ + '@phan-var-force Webklex\PHPIMAP\Message $imapemail'; if ($imapemail->hasHTMLBody()) { $htmlmsg = $imapemail->getHTMLBody(); } @@ -1972,7 +1972,21 @@ class EmailCollector extends CommonObject $to = $overview['to']; $sendtocc = empty($overview['cc']) ? '' : $overview['cc']; $sendtobcc = empty($overview['bcc']) ? '' : $overview['bcc']; - $date = $overview['date']; + + $tmpdate = $overview['date']->toDate(); // @phan-suppress-current-line PhanPluginUnknownObjectMethodCall + $tmptimezone = $tmpdate->getTimezone()->getName(); // @phan-suppress-current-line PhanPluginUnknownObjectMethodCall + + $dateemail = dol_stringtotime((string) $overview['date'], 'gmt'); // if $overview['timezone'] is "+00:00" + if (preg_match('/^([+\-])(\d\d):(\d\d)/', $tmptimezone, $reg)) { + if ($reg[1] == '+' && ($reg[2] != '00' || $reg[3] != '00')) { + $dateemail -= (3600 * (int) $reg[2]); + $dateemail -= (60 * (int) $reg[3]); + } + if ($reg[1] == '-' && ($reg[2] != '00' || $reg[3] != '00')) { + $dateemail += (3600 * (int) $reg[2]); + $dateemail += (60 * (int) $reg[3]); + } + } $subject = $overview['subject']; } else { $fromstring = $overview[0]->from; @@ -1982,7 +1996,7 @@ class EmailCollector extends CommonObject $to = $overview[0]->to; $sendtocc = !empty($overview[0]->cc) ? $overview[0]->cc : ''; $sendtobcc = !empty($overview[0]->bcc) ? $overview[0]->bcc : ''; - $date = $overview[0]->udate; + $dateemail = dol_stringtotime((string) $overview[0]->udate, 'gmt'); $subject = $overview[0]->subject; //var_dump($msgid);exit; } @@ -2045,7 +2059,7 @@ class EmailCollector extends CommonObject // var_dump($arrayofreferences); foreach ($arrayofreferences as $reference) { - //print "Process mail ".$iforemailloop." email_msgid ".$msgid.", date ".dol_print_date($date, 'dayhour').", subject ".$subject.", reference ".dol_escape_htmltag($reference)."
\n"; + //print "Process mail ".$iforemailloop." email_msgid ".$msgid.", date ".dol_print_date($dateemail, 'dayhour', 'gmt').", subject ".$subject.", reference ".dol_escape_htmltag($reference)."
\n"; if (!empty($trackidfoundintorecipienttype)) { $resultsearchtrackid = -1; // trackid found $reg[1] = $trackidfoundintorecipienttype; @@ -2218,7 +2232,7 @@ class EmailCollector extends CommonObject } else { foreach ($arrayofreferences as $key => $referencetmp) { if (!str_contains($objectemail->origin_references, $referencetmp)) { - $objectemail->origin_references.= " ".$referencetmp; + $objectemail->origin_references .= " ".$referencetmp; $changeonticket_references = true; } } @@ -2355,11 +2369,20 @@ class EmailCollector extends CommonObject dol_syslog("Execute action ".$operation['type']." actionparam=".$operation['actionparam'].' thirdpartystatic->id='.$thirdpartystatic->id.' contactstatic->id='.$contactstatic->id.' projectstatic->id='.$projectstatic->id); dol_syslog("Execute action fk_element_id=".$fk_element_id." fk_element_type=".$fk_element_type); // If a Dolibarr tracker id is found, we should now the id of object + // Try to guess if this is an email in or out. $actioncode = 'EMAIL_IN'; // If we scan the Sent box, we use the code for out email - if (preg_match('/Sent$/', $sourcedir)) { - $actioncode = 'EMAIL_OUT'; + if (preg_match('/Sent$/', $sourcedir) || preg_match('/envoyés$/i', $sourcedir)) { + $actioncode = 'EMAIL'; } + // If sender is in the list MAIL_FROM_EMAILS_TO_CONSIDER_SENDING + $arrayofemailtoconsideresender = explode(',', getDolGlobalString('MAIL_FROM_EMAILS_TO_CONSIDER_SENDING')); + foreach ($arrayofemailtoconsideresender as $emailtoconsidersender) { + if (preg_match('/'.preg_quote($emailtoconsidersender, '/').'/', $fromstring)) { + $actioncode = 'EMAIL'; + } + } + $operationslog .= '
Email will have actioncode='.$actioncode; $description = $descriptiontitle = $descriptionmeta = $descriptionfull = ''; @@ -2367,7 +2390,7 @@ class EmailCollector extends CommonObject $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("EmailMsgID").' : '.dol_escape_htmltag($msgid)); $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("MailTopic").' : '.dol_escape_htmltag($subject)); - $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("MailDate").($langs->trans("MailDate") != 'Date' ? ' (Date)' : '').' : '.dol_escape_htmltag(dol_print_date($date, "dayhourtext"))); + $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("MailDate").($langs->trans("MailDate") != 'Date' ? ' (Date)' : '').' : '.dol_escape_htmltag(dol_print_date($dateemail, "dayhourtext", "gmt"))); $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("MailFrom").($langs->trans("MailFrom") != 'From' ? ' (From)' : '').' : '.dol_escape_htmltag($fromstring)); if ($sender) { $descriptionmeta = dol_concatdesc($descriptionmeta, $langs->trans("Sender").($langs->trans("Sender") != 'Sender' ? ' (Sender)' : '').' : '.dol_escape_htmltag($sender)); @@ -2555,7 +2578,7 @@ class EmailCollector extends CommonObject dol_syslog("Third party with id=".$idtouseforthirdparty." email=".$emailtouseforthirdparty." name=".$nametouseforthirdparty." name_alias=".$namealiastouseforthirdparty." was not found"); //search into contacts of thirdparty - $resultContact = $contactstatic->fetch('', '', '', $emailtouseforthirdparty); + $resultContact = $contactstatic->fetch(0, '', '', $emailtouseforthirdparty); if ($resultContact > 0) { $idtouseforthirdparty = $contactstatic->socid; $result = $thirdpartystatic->fetch($idtouseforthirdparty); @@ -2657,7 +2680,7 @@ class EmailCollector extends CommonObject // Search country by name or code if (!empty($contactstatic->country)) { require_once DOL_DOCUMENT_ROOT . '/core/lib/company.lib.php'; - $result = getCountry('', 3, $this->db, '', 1, $contactstatic->country); + $result = getCountry('', 3, $this->db, null, 1, $contactstatic->country); if ($result == 'NotDefined') { $errorforactions++; $this->error = "Error country not found by this name '" . $contactstatic->country . "'"; @@ -2774,8 +2797,8 @@ class EmailCollector extends CommonObject $actioncomm->label = $langs->trans("ActionAC_".$actioncode).' - '.$langs->trans("MailFrom").' '.$from; $actioncomm->note_private = $descriptionfull; $actioncomm->fk_project = $projectstatic->id; - $actioncomm->datep = $date; // date of email - $actioncomm->datef = $date; // date of email + $actioncomm->datep = $dateemail; // date of email + $actioncomm->datef = $dateemail; // date of email $actioncomm->percentage = -1; // Not applicable $actioncomm->socid = $thirdpartystatic->id; $actioncomm->contact_id = $contactstatic->id; @@ -2826,7 +2849,7 @@ class EmailCollector extends CommonObject $operationslog .= '
Ticket Re-Opened successfully -> ref='.$objectemail->ref; } else { $errorforactions++; - $this->error = 'Error while changing the tcket status -> ref='.$objectemail->ref; + $this->error = 'Error while changing the ticket status -> ref='.$objectemail->ref; $this->errors[] = $this->error; } } @@ -3063,7 +3086,7 @@ class EmailCollector extends CommonObject $percent_opp_status = dol_getIdFromCode($this->db, 'PROSP', 'c_lead_status', 'code', 'percent'); $projecttocreate->title = $subject; - $projecttocreate->date_start = $date; // date of email + $projecttocreate->date_start = $dateemail; // date of email $projecttocreate->date_end = 0; $projecttocreate->opp_status = $id_opp_status; $projecttocreate->opp_percent = $percent_opp_status; @@ -3082,7 +3105,7 @@ class EmailCollector extends CommonObject if (empty($projecttocreate->ref)) { // Get next Ref $defaultref = ''; - $modele = !getDolGlobalString('PROJECT_ADDON') ? 'mod_project_simple' : $conf->global->PROJECT_ADDON; + $modele = getDolGlobalString('PROJECT_ADDON', 'mod_project_simple'); // Search template files $file = ''; @@ -3111,6 +3134,7 @@ class EmailCollector extends CommonObject $result = dol_include_once($reldir."core/modules/project/".$modele.'.php'); $modModuleToUseForNextValue = new $classname(); + '@phan-var-force ModeleNumRefProjects $modModuleToUseForNextValue'; $defaultref = $modModuleToUseForNextValue->getNextValue(($thirdpartystatic->id > 0 ? $thirdpartystatic : null), $projecttocreate); } $projecttocreate->ref = $defaultref; @@ -3209,7 +3233,7 @@ class EmailCollector extends CommonObject $tickettocreate->note_private = $descriptionfull; $tickettocreate->entity = $conf->entity; $tickettocreate->email_msgid = $msgid; - $tickettocreate->email_date = $date; + $tickettocreate->email_date = $dateemail; //$tickettocreate->fk_contact = $contactstatic->id; $savesocid = $tickettocreate->socid; @@ -3251,6 +3275,7 @@ class EmailCollector extends CommonObject $result = dol_include_once($reldir."core/modules/ticket/".$modele.'.php'); $modModuleToUseForNextValue = new $classname(); + '@phan-var-force ModeleNumRefTicket $modModuleToUseForNextValue'; $defaultref = $modModuleToUseForNextValue->getNextValue(($thirdpartystatic->id > 0 ? $thirdpartystatic : null), $tickettocreate); } $tickettocreate->ref = $defaultref; @@ -3325,7 +3350,7 @@ class EmailCollector extends CommonObject $candidaturetocreate->note_private = $descriptionfull; $candidaturetocreate->entity = $conf->entity; $candidaturetocreate->email_msgid = $msgid; - $candidaturetocreate->email_date = $date; // date of email + $candidaturetocreate->email_date = $dateemail; // date of email $candidaturetocreate->status = $candidaturetocreate::STATUS_DRAFT; //$candidaturetocreate->fk_contact = $contactstatic->id; @@ -3441,25 +3466,10 @@ class EmailCollector extends CommonObject if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { // Move mail using PHP-IMAP dol_syslog("EmailCollector::doCollectOneCollector move message ".($imapemail->getHeader()->get('subject'))." to ".$targetdir, LOG_DEBUG); + $operationslog .= '
Move mail '.($this->uidAsString($imapemail)).' - '.$msgid.' - '.$imapemail->getHeader()->get('subject').' to '.$targetdir; - if (empty($mode)) { // $mode > 0 is test - $operationslog .= '
Move mail '.($this->uidAsString($imapemail)).' - '.$msgid.' to '.$targetdir; - - $tmptargetdir = $targetdir; - if (!getDolGlobalString('MAIL_DISABLE_UTF7_ENCODE_OF_DIR')) { - $tmptargetdir = $this->getEncodedUtf7($targetdir); - } - - $result = $imapemail->move($tmptargetdir); - - if (empty($result)) { - dol_syslog("Failed to move email into target directory ".$targetdir); - $operationslog .= '
Failed to move email into target directory '.$targetdir; - $errorforemail++; - } - } else { - $operationslog .= '
Do not move mail '.($this->uidAsString($imapemail)).' - '.$msgid.' (test mode)'; - } + $arrayofemailtodelete[$this->uidAsString($imapemail)] = $imapemail; + // Note: Real move is done later using $arrayofemailtodelete } else { dol_syslog("EmailCollector::doCollectOneCollector move message ".($this->uidAsString($imapemail))." to ".$connectstringtarget, LOG_DEBUG); $operationslog .= '
Move mail '.($this->uidAsString($imapemail)).' - '.$msgid; @@ -3469,7 +3479,7 @@ class EmailCollector extends CommonObject } } else { if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { - dol_syslog("EmailCollector::doCollectOneCollector message '".($imapemail->getHeader()->get('subject'))."' using this->host=".$this->host.", this->access_type=".$this->acces_type." was set to read", LOG_DEBUG); + dol_syslog("EmailCollector::doCollectOneCollector message ".($this->uidAsString($imapemail))." '".($imapemail->getHeader()->get('subject'))."' using this->host=".$this->host.", this->access_type=".$this->acces_type." was set to read", LOG_DEBUG); } else { dol_syslog("EmailCollector::doCollectOneCollector message ".($this->uidAsString($imapemail))." to ".$connectstringtarget." was set to read", LOG_DEBUG); } @@ -3519,16 +3529,52 @@ class EmailCollector extends CommonObject // Disconnect if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { + // We revert the order to move/delete the more recent first (with higher number) so renumbering does not affect number of others to delete + $arrayofemailtodelete = array_reverse($arrayofemailtodelete, true); + + foreach ($arrayofemailtodelete as $imapemailnum => $imapemail) { + dol_syslog("EmailCollect::doCollectOneCollector delete email ".$imapemailnum); + + $operationslog .= "
move email ".$imapemailnum.($mode> 0 ? ' (test)' : ''); + + if (empty($mode) && empty($error)) { + $tmptargetdir = $targetdir; + if (!getDolGlobalString('MAIL_DISABLE_UTF7_ENCODE_OF_DIR')) { + $tmptargetdir = $this->getEncodedUtf7($targetdir); + } + + $result = 0; + try { + $result = $imapemail->move($tmptargetdir, false); + } catch (Exception $e) { + // Nothing to do. $result will remain 0 + $operationslog .= '
Exception !!!! '.$e->getMessage(); + } + if (empty($result)) { + dol_syslog("Failed to move email into target directory ".$targetdir); + $operationslog .= '
Failed to move email into target directory '.$targetdir; + $error++; + } + } + } + + if (empty($mode) && empty($error)) { + dol_syslog("Expunge", LOG_DEBUG); + $operationslog .= "
Expunge"; + + $client->expunge(); // To validate all moves + } + $client->disconnect(); } else { foreach ($arrayofemailtodelete as $imapemail => $msgid) { dol_syslog("EmailCollect::doCollectOneCollector delete email ".$imapemail." ".$msgid); - $operationslog .= "
delete email ".$imapemail." ".$msgid; + $operationslog .= "
delete email ".$imapemail." ".$msgid.($mode> 0 ? ' (test)' : ''); if (empty($mode) && empty($error)) { $res = imap_mail_move($connection, $imapemail, $targetdir, CP_UID); - if ($res == false) { + if (!$res) { // $errorforemail++; // Not in loop, not needed, not initialised $this->error = imap_last_error(); $this->errors[] = $this->error; From 85d20800815393069cdc3542c37d492098d92ce9 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Wed, 28 Aug 2024 12:21:39 +0200 Subject: [PATCH 4/4] Debug emailcollector - finding thirdparty --- .../class/emailcollector.class.php | 116 ++++++++++++------ htdocs/societe/class/societe.class.php | 39 ++++++ 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/htdocs/emailcollector/class/emailcollector.class.php b/htdocs/emailcollector/class/emailcollector.class.php index 342bb9cd8cf..0794a80165c 100644 --- a/htdocs/emailcollector/class/emailcollector.class.php +++ b/htdocs/emailcollector/class/emailcollector.class.php @@ -1151,16 +1151,16 @@ class EmailCollector extends CommonObject if (array_key_exists($keyforsupportedoauth2array, $supportedoauth2array) && array_key_exists('name', $supportedoauth2array[$keyforsupportedoauth2array]) && !empty($supportedoauth2array[$keyforsupportedoauth2array]['name'])) { - $OAUTH_SERVICENAME = $supportedoauth2array[$keyforsupportedoauth2array]['name'].(!empty($keyforprovider) ? '-'.$keyforprovider : ''); + $OAUTH_SERVICENAME = $supportedoauth2array[$keyforsupportedoauth2array]['name'].(!empty($keyforprovider) ? '-'.$keyforprovider : ''); } - require_once DOL_DOCUMENT_ROOT.'/includes/OAuth/bootstrap.php'; - //$debugtext = "Host: ".$this->host."
Port: ".$this->port."
Login: ".$this->login."
Password: ".$this->password."
access type: ".$this->acces_type."
oauth service: ".$this->oauth_service."
Max email per collect: ".$this->maxemailpercollect; - //dol_syslog($debugtext); + require_once DOL_DOCUMENT_ROOT.'/includes/OAuth/bootstrap.php'; + //$debugtext = "Host: ".$this->host."
Port: ".$this->port."
Login: ".$this->login."
Password: ".$this->password."
access type: ".$this->acces_type."
oauth service: ".$this->oauth_service."
Max email per collect: ".$this->maxemailpercollect; + //dol_syslog($debugtext); - $token = ''; + $token = ''; - $storage = new DoliStorage($db, $conf, $keyforprovider); + $storage = new DoliStorage($db, $conf, $keyforprovider); try { $tokenobj = $storage->retrieveAccessToken($OAUTH_SERVICENAME); @@ -1178,7 +1178,7 @@ class EmailCollector extends CommonObject getDolGlobalString('OAUTH_'.$this->oauth_service.'_ID'), getDolGlobalString('OAUTH_'.$this->oauth_service.'_SECRET'), getDolGlobalString('OAUTH_'.$this->oauth_service.'_URLCALLBACK') - ); + ); $serviceFactory = new \OAuth\ServiceFactory(); $oauthname = explode('-', $OAUTH_SERVICENAME); // ex service is Google-Emails we need only the first part Google @@ -1204,17 +1204,17 @@ class EmailCollector extends CommonObject return -1; } - $cm = new ClientManager(); - $client = $cm->make([ - 'host' => $this->host, - 'port' => $this->port, - 'encryption' => !empty($this->imap_encryption) ? $this->imap_encryption : false, - 'validate_cert' => true, - 'protocol' => 'imap', - 'username' => $this->login, - 'password' => $token, - 'authentication' => "oauth", - ]); + $cm = new ClientManager(); + $client = $cm->make([ + 'host' => $this->host, + 'port' => $this->port, + 'encryption' => !empty($this->imap_encryption) ? $this->imap_encryption : false, + 'validate_cert' => true, + 'protocol' => 'imap', + 'username' => $this->login, + 'password' => $token, + 'authentication' => "oauth", + ]); } else { // Mode LOGIN (login/pass) with PHP-IMAP $this->debuginfo .= 'doCollectOneCollector is using method MAIN_IMAP_USE_PHPIMAP=1, access_type=0 (LOGIN)
'; @@ -1402,7 +1402,7 @@ class EmailCollector extends CommonObject } } - if (empty($targetdir) || !getDolGlobalString('EMAILCOLLECTOR_NO_FILTER_ON_DATE_IF_THERE_IS_A_TARGETDIR')) { // Use last date as filter if there is no targetdir defined. + if (empty($targetdir) || !getDolGlobalString('EMAILCOLLECTOR_NO_FILTER_ON_DATE_IF_THERE_IS_A_TARGETDIR')) { // Use the last date of successful check as a filter if there is no targetdir defined. $fromdate = 0; if ($this->datelastok) { $fromdate = $this->datelastok; @@ -1656,13 +1656,12 @@ class EmailCollector extends CommonObject */ dol_syslog("Start of loop on email", LOG_INFO, 1); - $iforemailloop = 0; + $richarrayofemail = array(); + foreach ($arrayofemail as $imapemail) { if ($nbemailprocessed > 1000) { break; // Do not process more than 1000 email per launch (this is a different protection than maxnbcollectedpercollect) } - $iforemailloop++; - // GET header and overview datas if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { @@ -1680,6 +1679,24 @@ class EmailCollector extends CommonObject preg_match_all('/([^: ]+): (.+?(?:\r\n\s(?:.+?))*)\r\n/m', $header, $matches); $headers = array_combine($matches[1], $matches[2]); + + $richarrayofemail[] = array('imapemail' => $imapemail, 'header' => $header, 'headers' => $headers, 'overview' => $overview, 'date' => strtotime($headers['Date'])); + } + + + // Sort email found by ascending date + $richarrayofemail = dol_sort_array($richarrayofemail, 'date', 'asc'); + + + $iforemailloop = 0; + foreach ($richarrayofemail as $tmpval) { + $iforemailloop++; + + $imapemail = $tmpval['imapemail']; + $header = $tmpval['header']; + $overview = $tmpval['overview']; + $headers = $tmpval['headers']; + if (!empty($headers['in-reply-to']) && empty($headers['In-Reply-To'])) { $headers['In-Reply-To'] = $headers['in-reply-to']; } @@ -2136,11 +2153,11 @@ class EmailCollector extends CommonObject $objectemail = new Adherent($this->db); } /*if ($reg[1] == 'leav') { // Leave / Holiday - $objectemail = new Holiday($db); - } - if ($reg[1] == 'exp') { // ExpenseReport - $objectemail = new ExpenseReport($db); - }*/ + $objectemail = new Holiday($db); + } + if ($reg[1] == 'exp') { // ExpenseReport + $objectemail = new ExpenseReport($db); + }*/ } elseif (preg_match('/<(.*@.*)>/', $reference, $reg)) { // This is an external reference, we check if we have it in our database if (is_null($objectemail) && isModEnabled('ticket')) { @@ -2219,7 +2236,7 @@ class EmailCollector extends CommonObject $projectid = $objectemail->id; } - if (get_class($objectemail) == 'Ticket') { + if ($objectemail instanceof Ticket) { $ticketid = $objectemail->id; $changeonticket_references = false; @@ -2238,7 +2255,7 @@ class EmailCollector extends CommonObject } } if ($changeonticket_references) { - $objectemail->update($user); + $objectemail->update($user, 1); // We complete the references field, that is a field for technical tracking purpose, not a user field, so no need to execute triggers } } } @@ -2566,14 +2583,15 @@ class EmailCollector extends CommonObject $operationslog .= '
We try to search existing thirdparty with '.$idtouseforthirdparty.' '.$emailtouseforthirdparty.' '.$nametouseforthirdparty.' '.$namealiastouseforthirdparty; - $result = $thirdpartystatic->fetch($idtouseforthirdparty, $nametouseforthirdparty, '', '', '', '', '', '', '', '', $emailtouseforthirdparty, $namealiastouseforthirdparty); - // TODO Replace with fetchBySimilarityRules() + // Try to find the thirdparty that match the most the information we have + $result = $thirdpartystatic->findNearest($idtouseforthirdparty, $nametouseforthirdparty, '', '', '', '', '', '', '', '', $emailtouseforthirdparty, $namealiastouseforthirdparty); + if ($result < 0) { $errorforactions++; $this->error = 'Error when getting thirdparty with name '.$nametouseforthirdparty.' (may be 2 record exists with same name ?)'; $this->errors[] = $this->error; break; - } elseif ($result == 0) { + } elseif ($result == 0) { // No thirdparty found if ($operation['type'] == 'loadthirdparty') { dol_syslog("Third party with id=".$idtouseforthirdparty." email=".$emailtouseforthirdparty." name=".$nametouseforthirdparty." name_alias=".$namealiastouseforthirdparty." was not found"); @@ -2637,6 +2655,8 @@ class EmailCollector extends CommonObject } else { dol_syslog("One and only one existing third party has been found"); + $thirdpartystatic->fetch($result); + $operationslog .= '
Thirdparty already exists with id = '.dol_escape_htmltag($thirdpartystatic->id); } } @@ -3093,7 +3113,16 @@ class EmailCollector extends CommonObject $projecttocreate->description = dol_concatdesc(dolGetFirstLineOfText(dol_string_nohtmltag($description, 2), 10), '...'.$langs->transnoentities("SeePrivateNote").'...'); $projecttocreate->note_private = $descriptionfull; $projecttocreate->entity = $conf->entity; - $projecttocreate->email_msgid = $msgid; + // Fields when action is an email (content should be added into agenda event) + $projecttocreate->email_date = $dateemail; + $projecttocreate->email_msgid = $msgid; + $projecttocreate->email_from = $fromstring; + $projecttocreate->email_sender = $sender; + $projecttocreate->email_to = $to; + $projecttocreate->email_tocc = $sendtocc; + $projecttocreate->email_tobcc = $sendtobcc; + $projecttocreate->email_subject = $subject; + $projecttocreate->errors_to = ''; $savesocid = $projecttocreate->socid; @@ -3232,8 +3261,17 @@ class EmailCollector extends CommonObject $tickettocreate->notify_tiers_at_create = getDolGlobalInt('TICKET_CHECK_NOTIFY_THIRDPARTY_AT_CREATION'); $tickettocreate->note_private = $descriptionfull; $tickettocreate->entity = $conf->entity; - $tickettocreate->email_msgid = $msgid; - $tickettocreate->email_date = $dateemail; + // Fields when action is an email (content should be added into agenda event) + $tickettocreate->email_date = $dateemail; + $tickettocreate->email_msgid = $msgid; + $tickettocreate->email_from = $fromstring; + $tickettocreate->email_sender = $sender; + $tickettocreate->email_to = $to; + $tickettocreate->email_tocc = $sendtocc; + $tickettocreate->email_tobcc = $sendtobcc; + $tickettocreate->email_subject = $subject; + $tickettocreate->errors_to = ''; + //$tickettocreate->fk_contact = $contactstatic->id; $savesocid = $tickettocreate->socid; @@ -3289,6 +3327,10 @@ class EmailCollector extends CommonObject $this->error = 'Failed to create ticket: Can\'t get a valid value for the field ref with numbering template = '.$modele.', thirdparty id = '.$thirdpartystatic->id; } else { // Create ticket + $tickettocreate->context['actionmsg2'] = $langs->trans("ActionAC_EMAIL_IN").' - '.$langs->trans("TICKET_CREATEInDolibarr"); + $tickettocreate->context['actionmsg'] = $langs->trans("ActionAC_EMAIL_IN").' - '.$langs->trans("TICKET_CREATEInDolibarr"); + //$tickettocreate->email_fields_no_propagate_in_actioncomm = 0; + $result = $tickettocreate->create($user); if ($result <= 0) { $errorforactions++; @@ -3529,8 +3571,8 @@ class EmailCollector extends CommonObject // Disconnect if (getDolGlobalString('MAIN_IMAP_USE_PHPIMAP')) { - // We revert the order to move/delete the more recent first (with higher number) so renumbering does not affect number of others to delete - $arrayofemailtodelete = array_reverse($arrayofemailtodelete, true); + // We sort to move/delete array with the more recent first (with higher number) so renumbering does not affect number of others to delete + krsort($arrayofemailtodelete, SORT_NUMERIC); foreach ($arrayofemailtodelete as $imapemailnum => $imapemail) { dol_syslog("EmailCollect::doCollectOneCollector delete email ".$imapemailnum); diff --git a/htdocs/societe/class/societe.class.php b/htdocs/societe/class/societe.class.php index 3bed18d74c8..637f231258f 100644 --- a/htdocs/societe/class/societe.class.php +++ b/htdocs/societe/class/societe.class.php @@ -2100,6 +2100,45 @@ class Societe extends CommonObject return $result; } + /** + * Search the thirdparty that match the most the provided parameters. + * Searching rules try to find the existing third party. + * + * @param int $rowid Id of third party + * @param string $ref Reference of third party, name (Warning, this can return several records) + * @param string $ref_ext External reference of third party (Warning, this information is a free field not provided by Dolibarr) + * @param string $barcode Barcode of third party to load + * @param string $idprof1 Prof id 1 of third party (Warning, this can return several records) + * @param string $idprof2 Prof id 2 of third party (Warning, this can return several records) + * @param string $idprof3 Prof id 3 of third party (Warning, this can return several records) + * @param string $idprof4 Prof id 4 of third party (Warning, this can return several records) + * @param string $idprof5 Prof id 5 of third party (Warning, this can return several records) + * @param string $idprof6 Prof id 6 of third party (Warning, this can return several records) + * @param string $email Email of third party (Warning, this can return several records) + * @param string $ref_alias Name_alias of third party (Warning, this can return several records) + * @param int $is_client Only client third party + * @param int $is_supplier Only supplier third party + * @return int >0 if OK, <0 if KO or if two records found, 0 if not found. + */ + public function findNearest($rowid = 0, $ref = '', $ref_ext = '', $barcode = '', $idprof1 = '', $idprof2 = '', $idprof3 = '', $idprof4 = '', $idprof5 = '', $idprof6 = '', $email = '', $ref_alias = '', $is_client = 0, $is_supplier = 0) + { + // A rowid is known, it is a unique key so we found it + if ($rowid) { + return $rowid; + } + + // We try to find the thirdparty with exact matching on all fields + // TODO Replace this with step by step search + // Then search on barcode if we have it (+ restriction on is_client and is_supplier) + // Then search on profids with a OR (+ restriction on is_client and is_supplier) + // Then search on email with a OR (+ restriction on is_client and is_supplier) + // Then search ref, ref_ext or alias with a OR (+ restriction on is_client and is_supplier) + $tmpthirdparty = new Societe($this->db); + $result = $tmpthirdparty->fetch($rowid, $ref, $ref_ext, $barcode, $idprof1, $idprof2, $idprof3, $idprof4, $idprof5, $idprof6, $email, $ref_alias, $is_client, $is_supplier); + + return $result; + } + /** * Delete a third party from database and all its dependencies (contacts, rib...) *