From b91c2f5f76fb442d207a7a75775c656a0f64415e Mon Sep 17 00:00:00 2001 From: Mohamed DAOUD Date: Tue, 25 Feb 2025 17:44:13 +0100 Subject: [PATCH] manage community repository as a source --- htdocs/admin/modules.php | 76 ++-- .../class/externalModules.class.php | 373 +++++++++++++++++- 2 files changed, 411 insertions(+), 38 deletions(-) diff --git a/htdocs/admin/modules.php b/htdocs/admin/modules.php index 7215d207ff2..83b7e7f8db8 100644 --- a/htdocs/admin/modules.php +++ b/htdocs/admin/modules.php @@ -76,18 +76,27 @@ $action = GETPOST('action', 'aZ09'); $value = GETPOST('value', 'alpha'); $page_y = GETPOSTINT('page_y'); $search_keyword = GETPOST('search_keyword', 'alpha'); +$search_source_dolistore = GETPOSTINT('search_source_dolistore'); +$search_source_github = GETPOSTINT('search_source_github'); $search_status = GETPOST('search_status', 'alpha'); $search_nature = GETPOST('search_nature', 'alpha'); $search_version = GETPOST('search_version', 'alpha'); // For remotestore search -$options = array(); -$options['per_page'] = 10; -$options['no_page'] = ((int) GETPOSTINT('no_page') ? GETPOSTINT('no_page') : 1); -$options['categorie'] = ((int) (GETPOSTINT('categorie') ? GETPOSTINT('categorie') : 0)); -$options['search'] = GETPOST('search_keyword', 'alpha'); -$options['source'] = GETPOST('source', 'alpha'); +$options = array(); +$options['per_page'] = 10; +$options['no_page'] = ((int) GETPOSTINT('no_page') ? GETPOSTINT('no_page') : 1); +$options['categorie'] = ((int) (GETPOSTINT('categorie') ? GETPOSTINT('categorie') : 0)); +$options['search'] = GETPOST('search_keyword', 'alpha'); +$options['search_source_dolistore'] = GETPOSTINT('search_source_dolistore'); +$options['search_source_github'] = GETPOSTINT('search_source_github'); + +// Set default values +if (empty($options['search_source_dolistore']) && empty($options['search_source_github'])) { + $options['search_source_dolistore'] = 1; + $options['search_source_github'] = 1; +} //$remotestore = new Dolistore(false); $remotestore = new ExternalModules(); @@ -116,6 +125,12 @@ if (!GETPOST('buttonreset', 'alpha')) { if ($search_keyword) { $param .= '&search_keyword='.urlencode($search_keyword); } + if ($search_source_dolistore) { + $param .= '&search_source_dolistore='.urlencode($search_source_dolistore); + } + if ($search_source_github) { + $param .= '&search_source_github='.urlencode($search_source_github); + } if ($search_status && $search_status != '-1') { $param .= '&search_status='.urlencode($search_status); } @@ -175,6 +190,8 @@ if (GETPOST('buttonreset', 'alpha')) { $search_status = ''; $search_nature = ''; $search_version = ''; + $search_source_dolistore = ''; + $search_source_github = ''; } if ($action == 'install' && $allowonlineinstall) { @@ -1254,6 +1271,7 @@ if ($mode == 'marketplace') { print ''.$form->textwithpicto($langs->trans("Provider"), $langs->trans("WebSiteDesc")).''; print ''; print ''.$langs->trans("URL").''; + print ''; print ''; // Marketplace @@ -1262,6 +1280,7 @@ if ($mode == 'marketplace') { print ''; print ''.$langs->trans("DoliStoreDesc").''; print ''.$url.''; + print '' . $remotestore->libStatut($remotestore->dolistoreApiStatus).''; print ''; // Community @@ -1270,6 +1289,7 @@ if ($mode == 'marketplace') { print ''; print ''.$langs->trans("CommunityModulesDesc").''; print ''.$url.''; + print '' . $remotestore->libStatut($remotestore->githubFileStatus) . ''; print ''; print "\n"; @@ -1279,9 +1299,9 @@ if ($mode == 'marketplace') { print '
'; - $conf->global->MAIN_DISABLE_DOLISTORE_SEARCH = 1; // avoid warning with the new Dolistore website + $conf->global->MAIN_DISABLE_DOLISTORE_SEARCH = 0; // avoid warning with the new Dolistore website - if (!getDolGlobalString('MAIN_DISABLE_DOLISTORE_SEARCH') && getDolGlobalInt('MAIN_FEATURES_LEVEL') >= 2) { + if (!getDolGlobalString('MAIN_DISABLE_DOLISTORE_SEARCH') && getDolGlobalInt('MAIN_FEATURES_LEVEL') >= 2 && $remotestore->numberOfProviders > 0) { // $options is array with filter criteria if (getDolGlobalInt('MAIN_ENANLE_OLD_DOLISTORE')) { @@ -1308,12 +1328,23 @@ if ($mode == 'marketplace') {
- - -
+ + + dolistoreApiStatus > 0 && $remotestore->numberOfProviders > 1) : ?> + > + + + + githubFileStatus > 0 && $remotestore->numberOfProviders > 1) : ?> + > + + +
@@ -1330,15 +1361,16 @@ if ($mode == 'marketplace') { print '
'; print '
'; ?> + +
+ +
+ -
- -
- -
+
style="width:100%;"> get_products($nbmaxtoshow); ?> diff --git a/htdocs/admin/remotestore/class/externalModules.class.php b/htdocs/admin/remotestore/class/externalModules.class.php index beb31d32c46..993df272ba1 100644 --- a/htdocs/admin/remotestore/class/externalModules.class.php +++ b/htdocs/admin/remotestore/class/externalModules.class.php @@ -46,6 +46,17 @@ class ExternalModules /** * @var string */ + + /** + * @var string GitHub YAML file URL + */ + public $file_source_url; + + /** + * @var string Cache file path for GitHub modules YAML file content (local) + */ + public $cache_file; + public $url; // the url of this page /** * @var string @@ -68,6 +79,26 @@ class ExternalModules */ public $dolistore_api_key; + /** + * @var bool + */ + public $dolistoreApiStatus; + + /** + * @var string + */ + public $dolistoreApiError; + + /** + * @var bool + */ + public $githubFileStatus; + + /** + * @var int // number of online providers + */ + public $numberOfProviders; + /** * @var array|null */ @@ -89,6 +120,10 @@ class ExternalModules $this->shop_url = 'https://www.dolistore.com/product.php?id='; $this->debug_api = $debug; + $this->file_source_url = "https://raw.githubusercontent.com/Dolibarr/dolibarr-community-modules/refs/heads/main/index.yaml"; + $this->cache_file = DOL_DOCUMENT_ROOT.'/admin/remotestore/sources/github_modules_file.yaml'; + $this->getRemoteYamlFile($this->file_source_url, 86400); + $lang = $langs->defaultlang; $lang_array = array('en_US', 'fr_FR', 'es_ES', 'it_IT', 'de_DE'); if (!in_array($lang, $lang_array)) { @@ -96,7 +131,12 @@ class ExternalModules } $this->lang = $lang; - // TODO check access to all remote sources and define valid one + // Check access to Dolistore API + $this->dolistoreApiStatus = $this->checkApiStatus(); + $this->githubFileStatus = file_exists($this->cache_file); + + // Count the number of online providers + $this->numberOfProviders = $this->dolistoreApiStatus + $this->githubFileStatus; } /** @@ -105,7 +145,7 @@ class ExternalModules * @param string $resource Resource name * @param array|false $options Options for the request * - * @return array{status_code:int,response:?string,header:string} + * @return array{status_code:int,response:null|string|array,header:string} */ public function callApi($resource, $options = false) { @@ -139,6 +179,8 @@ class ExternalModules curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($curl, CURLOPT_HTTPHEADER, $httpheader); curl_setopt($curl, CURLOPT_HEADER, true); + curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, false); + curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, false); $response = curl_exec($curl); @@ -161,6 +203,30 @@ class ExternalModules return array('status_code' => $status_code, 'response' => $body, 'header' => $header); } + /** + * Fetch modules from a cache YAML file + * @param array $options Options for filter + * + * @return array List of modules + */ + public function fetchModulesFromFile($options = array()) + { + $modules = array(); + + if (!empty($this->cache_file) && file_exists($this->cache_file)) { + dol_syslog(__METHOD__ . " - Loading cache file: " . $this->cache_file, LOG_DEBUG); + + $content = file_get_contents($this->cache_file); + if ($content !== false) { + $modules = $this->readYaml($content); + } else { + dol_syslog(__METHOD__ . " - Error reading cache file", LOG_ERR); + } + } + + return $modules; + } + /** * Generate HTML for categories and their children. * @@ -182,8 +248,6 @@ class ExternalModules return $html ; } - // TODO call all sources and merge them - $html = ''; foreach ($organized_tree as $key => $value) { if ($value['label'] != "Versions" && $value['label'] != "Specials") { @@ -215,12 +279,9 @@ class ExternalModules global $langs; - // TODO call all sources and merge them - $html = ""; $last_month = dol_now() - (30 * 24 * 60 * 60); $dolibarrversiontouse = DOL_VERSION; - $urldolibarrmodules = 'https://www.dolistore.com'; $this->products = array(); @@ -237,14 +298,32 @@ class ExternalModules 'lang' => $this->lang ]; - $resProducts = $this->callApi('products', $data); - if (!isset($resProducts['response']) || !is_array($resProducts['response']) || ($resProducts['status_code'] != 200 && $resProducts['status_code'] != 201)) { - $html = $this->checkStatusCode($resProducts); - - return $html; + // Fetch the products from Dolistore source + $dolistoreProducts = array(); + if ($this->dolistoreApiStatus > 0 && $options['search_source_dolistore'] == 1) { + $getDolistoreProducts = $this->callApi('products', $data); + if (!isset($getDolistoreProducts['response']) || !is_array($getDolistoreProducts['response']) || ($getDolistoreProducts['status_code'] != 200 && $getDolistoreProducts['status_code'] != 201)) { + $dolistoreProducts = array(); + } else { + $dolistoreProducts = $this->adaptData($getDolistoreProducts['response'], 'dolistore'); + } } - $this->products = $resProducts['response']; + // fetch from github repo + $fileProducts = array(); + if (!empty($this->githubFileStatus) && $options['search_source_github'] == 1) { + $fileProducts = $this->fetchModulesFromFile($data); + $fileProducts = $this->adaptData($fileProducts, 'github'); + $fileProducts = $this->applyFilters($fileProducts, $data); + } + + // merge both sources + $this->products = array_merge($dolistoreProducts, $fileProducts); + + // Sort products list by datec + usort($this->products, function ($a, $b) { + return strtotime($b['datec'] ?? '0') - strtotime($a['datec'] ?? '0'); + }); $i = 0; foreach ($this->products as $product) { @@ -274,9 +353,17 @@ class ExternalModules $price = '

'.price(price2num($product["price_ttc"], 'MT'), 0, $langs, 1, -1, -1, 'EUR').' '.$langs->trans("TTC").'

'; $download_link = ''; } else { + $download_link = '#'; $price = '

'.$langs->trans('Free').'

'; - $download_link = ''; - $download_link .= ''; + if ($product['source'] === 'Dolistore') { + $download_link = ''; + $download_link .= ''; + } + + if ($product['source'] === 'Github') { + $download_link = ''; + $download_link .= ''; + } } // Set and check version @@ -322,7 +409,8 @@ class ExternalModules $html .= '
'; $html .= $version; // No dol_escape_htmltag, it is already escape html $html .= ''; - $html .= ' '.dol_print_date(dol_stringtotime($product['tms']), 'dayhour').' - '.$langs->trans('Ref').': '.dol_escape_htmltag($product["ref"]).' - '.dol_escape_htmltag($langs->trans('Id')).': '.((int) $product["id"]).'
'; + $html .= ' '.dol_print_date(dol_stringtotime($product['tms']), 'day').' - '.$langs->trans('Ref').': '.dol_escape_htmltag($product["ref"]).' - '.dol_escape_htmltag($langs->trans('Id')).': '.((int) $product["id"]).'
'; + $html .= ''.$langs->trans('Source').': '.$product["source"].'
'; $html .= '
'.dol_escape_htmltag(dol_string_nohtmltag($product["description"])); $html.= ''; // do not load if display none @@ -523,4 +611,257 @@ class ExternalModules // Return the formatted error message return sprintf('This call to the API failed and returned an HTTP status of %d. That means: %s.', $request['status_code'], $error_message); } + + /** + * get YAML file from remote source and put it in cache file for one day + * @param string $file_source_url URL of the remote source + * @param string $cache_time Cache time + * + * @return string Uri of the cache file + */ + public function getRemoteYamlFile($file_source_url, $cache_time) + { + $cache_file = $this->cache_file; + + // Check if cache directory exists + if (!file_exists(dirname($cache_file))) { + mkdir(dirname($cache_file), 0777, true); + } + + if (!file_exists($cache_file) || filemtime($cache_file) < dol_now() - $cache_time) { + $yaml = file_get_contents($file_source_url); + if (!empty($yaml)) { + file_put_contents($cache_file, $yaml); + } + } + + return $cache_file; + } + + + /** + * Read a YAML string and convert it to an array + * @param string $yaml YAML string + * @return array Parsed array representation + */ + public function readYaml($yaml) + { + $data = []; + $currentPackage = null; + $currentSection = null; + + foreach (explode("\n", trim($yaml)) as $line) { + $trimmedLine = trim($line); + + // Ignore empty lines and comments + if ($trimmedLine === '' || strpos($trimmedLine, '#') === 0) { + continue; + } + + // Match a new package entry (e.g., "- modulename: 'helloasso'") + if (preg_match('/^\s*-\s*modulename:\s*["\']?(.*?)["\']?$/', $trimmedLine, $matches)) { + if ($currentPackage !== null) { + $data[] = $currentPackage; + } + $currentPackage = ['modulename' => $matches[1]]; + $currentSection = null; + continue; + } + + // If the key doesn't start with fr, en, es, it, de, treat it as a section + if (!preg_match('/^\s*(fr|en|es|it|de):\s*["\']?(.*?)["\']?$/', $trimmedLine)) { + $currentSection = null; + } + + // Match a top-level key-value pair (e.g., "author: 'Dolicloud'") + if (preg_match('/^(\w[\w-]*):\s*["\']?(.*?)["\']?$/', $trimmedLine, $matches)) { + if ($currentPackage !== null) { + if ($currentSection) { + // Store in the nested section + $currentPackage[$currentSection][$matches[1]] = $matches[2] === '' ? null : $matches[2]; + } else { + // Store as a normal key-value pair + $currentPackage[$matches[1]] = $matches[2] === '' ? null : $matches[2]; + } + } + + // Match a nested section (e.g., "label:") + if (preg_match('/^\s*(label|description):\s*$/', $trimmedLine, $matches)) { + $currentSection = $matches[1]; + $currentPackage[$currentSection] = []; // Initialize as an empty array for nested sections + } + + continue; + } + } + + // Add the last package if available + if ($currentPackage !== null) { + $data[] = $currentPackage; + } + + return $data; + } + + /** + * Adapter data fetched from github remote source to the expected format + * @param array $data Data fetched from github remote source + * @param string $source Source of the data + * + * @return array Data adapted to the expected format + */ + public function adaptData($data, $source) + { + $adaptedData = []; + $urldolibarrmodules = 'https://www.dolistore.com'; + + if (!is_array($data) || empty($data) || empty($source)) { + return $adaptedData; + } + + if ($source === 'github') { + foreach ($data as $package) { + if (empty($package['modulename'])) { + continue; + } + $adaptedPackage = [ + 'ref' => str_replace(' ', '', $package['author'].'@'.$package['modulename'].'@'.$package['current_version']), + 'label' => !empty($package['label'][substr($this->lang, 0, 2)]) + ? $package['label'][substr($this->lang, 0, 2)] + : (!empty($package['label']['en']) ? $package['label']['en'] : $package['modulename']), + 'description' => !empty($package['description'][substr($this->lang, 0, 2)]) + ? $package['description'][substr($this->lang, 0, 2)] + : (!empty($package['description']['en']) ? $package['description']['en'] : ''), + 'datec' => !empty($package['created_at']) + ? date('Y-m-d H:i:s', strtotime($package['created_at'])) + : '', + 'tms' => !empty($package['last_updated_at']) + ? date('Y-m-d H:i:s', strtotime($package['last_updated_at'])) + : '', + 'price_ttc' => 0, + 'dolibarr_min' => !empty($package['dolibarrmin']) + ? $package['dolibarrmin'] + : 'unknown', + 'dolibarr_max' => !empty($package['dolibarrmax']) + ? $package['dolibarrmax'] + : 'unknown', + 'module_version' => !empty($package['current_version']) + ? $package['current_version'] + : 'unknown', + 'cover_photo_url' => !empty($package['cover']) + ? $package['cover'] + : '#', + 'category' => !empty($package['category']) + ? explode(',', str_replace(' ', '', $package['category'])) + : array(), + 'link' => !empty($package['git']) + ? $package['git'] + : '#', + 'source' => 'Github' + ]; + + $adaptedData[] = $adaptedPackage; + } + } + + if ($source === 'dolistore') { + foreach ($data as $package) { + $adaptedPackage = [ + 'id' => $package['id'], + 'ref'=> $package['ref'], + 'label' => $package['label'], + 'description' => $package['description'], + 'datec' => $package['datec'], + 'tms' => $package['tms'], + 'price_ttc' => $package['price_ttc'], + 'dolibarr_min' => $package['dolibarr_min'], + 'dolibarr_max' => $package['dolibarr_max'], + 'module_version' => $package['module_version'], + 'cover_photo_url' => $urldolibarrmodules.$package['cover_photo_url'], + 'source' => 'Dolistore' + ]; + + $adaptedData[] = $adaptedPackage; + } + } + + return $adaptedData; + } + + /** + * Apply filters to the data + * @param array $data Data to filter + * @param array $options Options for the filter + * + * @return array Filtered data + */ + public function applyFilters($data, $options) + { + $filteredData = $data; + + // Sort products list by datec + usort($filteredData, function ($a, $b) { + return strtotime($b['datec'] ?? '0') - strtotime($a['datec'] ?? '0'); + }); + + if (!empty($options['search'])) { + $filteredData = array_filter($filteredData, function ($package) use ($options) { + return stripos($package['label'], $options['search']) !== false || stripos($package['description'], $options['search']) !== false; + }); + } + + if (!empty($options['categorieid'])) { + $filteredData = array_filter($filteredData, function ($package) use ($options) { + return in_array($options['categorieid'], $package['category']); + }); + } + + // Pagination + $filteredData = array_values($filteredData); + $filteredData = array_slice($filteredData, ($options['page'] - 1) * $options['limit'], $options['limit']); + + return $filteredData; + } + + /** + * Check if an Dolistore API is up + * + * @return bool + */ + public function checkApiStatus() + { + + $testRequest = $this->callApi('categories'); + + if (!isset($testRequest['response']) || !is_array($testRequest['response']) || ($testRequest['status_code'] != 200 && $testRequest['status_code'] != 201)) { + $this->dolistoreApiError = $this->checkStatusCode($testRequest); + return 0; + } else { + return 1; + } + } + + /** + * Retrieve the status icon + * @param mixed $status Status + * @param mixed $mode Mode + * + * @return string + */ + public function libStatut($status, $mode = 3) + { + global $langs; + + $statusType = 'status4'; + if ($status == 0) { + $statusType = 'status6'; + } + + $labelStatus[0] = $this->dolistoreApiError; + $labelStatus[1] = $langs->transnoentitiesnoconv("online"); + $labelStatusShort[0] = $this->dolistoreApiError; + $labelStatusShort[1] = $langs->transnoentitiesnoconv("online"); + + return dolGetStatus($labelStatus[$status], $labelStatusShort[$status], '', $statusType, $mode); + } }