Extended messages cannot have required fields (must be optional).

resources/search.php

@@ -1,4 +1,4 @@
-<?
+<?php
 
 /* Copyright Camille Harang, Pablo Joubert, Emmanuel Benazera
 
@@ -15,18 +15,56 @@
 You should have received a copy of the GNU Affero General Public License
 along with this program. If not, see http://www.fsf.org/licensing/licenses/agpl-3.0.html. */
 
-if($_SERVER['HTTPS']) $scheme = 'https://';
-else $scheme= 'http://';
+/*****************************************************************************
+ * Important change to php.ini:                                              *
+ *****************************************************************************
+ * You MUST set always_populate_raw_post_data=1 in your php.ini to get this  *
+ * working!). E.g. I had to set this in FPM's seeks.conf:                    *
+ * php_admin_value[always_populate_raw_post_data] = 1                        *
+ *****************************************************************************
+ * My file is located at /etc/php5/fpm/pool.d/seeks.conf.                    *
+ *****************************************************************************
+ */
+
+// Let's fix all of them
+error_reporting(E_ALL | E_STRICT);
+
+// Default is HTTP
+$scheme = 'http://';
+
+// Is HTTPS set?
+if (isset($_SERVER['HTTPS'])) {
+	// Then assume https:// shall be used
+	$scheme = 'https://';
+}
 
 $seeks_uri = 'http://s.s';
 $proxy = 'localhost:8250';
-$base_script = $_SERVER['SCRIPT_NAME'];
-$base_url = $scheme.$_SERVER['HTTP_HOST'].$base_script;
 
-if ($_SERVER['REQUEST_URI'] == '/search.php') { header('Location: '.$base_url.'/websearch-hp'); }
-else $url = $seeks_uri.str_replace($base_script, '', $_SERVER['REQUEST_URI']);
-$lang_head = $_SERVER['HTTP_ACCEPT_LANGUAGE'];
-$referer = $_SERVER['HTTP_REFERER'];
+$base_url = $scheme . $_SERVER['HTTP_HOST'] . $_SERVER['SCRIPT_NAME'];
+
+if ($_SERVER['REQUEST_URI'] == '/search.php') {
+	header('Location: ' . $base_url . '/websearch-hp');
+
+	// Don't miss exit() as the server redirect doesn't abort the script
+	exit();
+} else {
+	$url = $seeks_uri . str_replace($_SERVER['SCRIPT_NAME'], '', $_SERVER['REQUEST_URI']);
+}
+
+// Avoid E_NOTICE by initializing variables and checking on array elements
+$lang_head = '';
+$referer = '';
+
+// Not all browsers are sending this header
+if (isset($_SERVER['HTTP_ACCEPT_LANGUAGE'])) {
+	$lang_head = $_SERVER['HTTP_ACCEPT_LANGUAGE'];
+}
+
+// Not all browsers are sending this header
+if (isset($_SERVER['HTTP_REFERER'])) {
+	$referer = $_SERVER['HTTP_REFERER'];
+}
 
 $qc_redir = array();
 preg_match('/qc_redir/', $url, $qc_redir);
@@ -39,43 +77,68 @@
 
 $curl = curl_init();
 curl_setopt($curl, CURLOPT_URL, $url);
-if ($qc_redir[0] == "qc_redir"
-   || $tbd[0] == "tbd")
-{
- curl_setopt($curl, CURLOPT_HEADER, true);
+
+if ((isset($qc_redir[0]) && $qc_redir[0] == 'qc_redir') || (isset($tbd[0]) && $tbd[0] == 'tbd')) {
+	curl_setopt($curl, CURLOPT_HEADER, true);
 }
+
 curl_setopt($curl, CURLOPT_CUSTOMREQUEST, $_SERVER['REQUEST_METHOD']);
-if ($bqc[0] == "find_bqc")
-{
- $postdata = file_get_contents("php://input");
- curl_setopt($curl, CURLOPT_POST, 1);
- curl_setopt($curl, CURLOPT_POSTFIELDS, $postdata);
+
+if (isset($bqc[0]) && $bqc[0] == 'find_bqc') {
+	$postdata = file_get_contents('php://input');
+
+	// For debugging purposes
+	//* DEBUG: */ file_put_contents('/tmp/foo', ini_get('always_populate_raw_post_data') . ':' . $postdata);
+
+	curl_setopt($curl, CURLOPT_POST, 1);
+	curl_setopt($curl, CURLOPT_POSTFIELDS, $postdata);
 }
+
 curl_setopt($curl, CURLOPT_PROXY, $proxy);
-curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1) ;
-curl_setopt($curl, CURLOPT_HTTPHEADER, array("Seeks-Remote-Location: ".$base_url,"Accept-Language: ".$lang_head,"Referer: " .$referer));
+curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
+
+curl_setopt($curl, CURLOPT_HTTPHEADER, array('Seeks-Remote-Location: ' . $base_url, 'Accept-Language: ' . $lang_head, 'Proxy-Connection: Close', 'Expect:', 'Referer: ' . $referer));
+
 $result = curl_exec($curl);
 $result_info = curl_getinfo($curl);
-if(curl_errno($curl)) echo 'CURL ERROR: '.curl_error($curl);
+
+// Very noisy in syslog:
+//* NOISY-DEBUG: */ syslog(LOG_INFO, 'result_info=' . json_encode($result_info));
+
+if (curl_errno($curl)) {
+	echo 'CURL ERROR: '.curl_error($curl);
+}
+
 curl_close($curl);
 
-header('Content-Type: '.$result_info['content_type']);
-
-if ($qc_redir[0] == "qc_redir"
-   || $tbd[0] == "tbd")
-{
- $status_code = array(); 
- preg_match('/\d\d\d/', $result, $status_code);
- switch( $status_code[0] ) {
-  case 302:
-  $location = array();
-  preg_match('/Location: (.*)/', $result, $location);
-  $location[0] = substr($location[0],10);
-  header("Location: ". $location[0]);
-  break; 
-  default:
- }
+if (headers_sent()) {
+	// Headers are already sent, cannot continue, maybe error?
+	exit();
 }
 
+header('Content-Type: ' . $result_info['content_type']);
+
+if ((isset($qc_redir[0]) && $qc_redir[0] == 'qc_redir') || (isset($tbd[0]) && $tbd[0] == 'tbd')) {
+	preg_match('/\d\d\d/', $result, $status_code);
+
+	switch( $status_code[0] ) {
+	case 302:
+		$location = array();
+		preg_match('/Location: (.*)/', $result, $location);
+		$location[0] = substr($location[0], 10);
+		header('Location: '. $location[0]);
+
+		// Don't miss exit() as the server redirect doesn't abort the script
+		exit();
+
+	default:
+		syslog(LOG_WARNING, 'Status code ' . $status_code[0] . ' found.');
+		break;
+	}
+}
+
+// Very noisy in syslog:
+//* NOISY-DEBUG: */ syslog(LOG_INFO, 'result=' . json_encode($result));
 echo $result;
+
 ?>

src/cli/Makefile.am

@@ -5,7 +5,7 @@ seeksclidatadir = $(datadir)/seeks/cli
 
 AM_CXXFLAGS=-Wall -g -pipe
 AM_CPPFLAGS=-I${srcdir} -I${srcdir}/../proxy -I${srcdir}/../utils -I${srcdir}/../lsh \
-            @CURL_CFLAGS@
+            -I${srcdir}/../plugins/udb_service @CURL_CFLAGS@
 
 seeksclilib_LIBRARIES=libseekscli.a
 libseekscli_a_SOURCES=cli.cpp ../proxy/encode.cpp ../utils/miscutil.cpp cli.h

src/plugins/cf/tests/ut-cf-sre.cpp

@@ -563,7 +563,7 @@ TEST_F(SRETest,recommendation_post_url_check_retrieve)
   http_response rsp;
   hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters
   = new hash_map<const char*,const char*,hash<const char*>,eqstr>();
-  miscutil::add_map_entry(parameters,"url",1,"http://www.seeks.fr/",1);
+  miscutil::add_map_entry(parameters,"url",1,"http://seeks-project.info/",1);
   miscutil::add_map_entry(parameters,"url-check",1,"0",1);
   miscutil::add_map_entry(parameters,"radius",1,"5",1);
   miscutil::add_map_entry(parameters,"output",1,"json",1);

src/plugins/cf/tests/ut-peer-list.cpp

@@ -81,6 +81,7 @@ TEST(WCFTest,cgi_peers)
   cf_configuration::_config = new cf_configuration("");
   cf_configuration::_config->_pl->add("seeks.fr",-1,"","bsn");
   cf_configuration::_config->_pl->add("seeks-project.info",-1,"/search.php","bsn");
+  // @TODO Wont' work, as search_exp.php is not available?
   cf_configuration::_config->_pl->add("seeks-project.info",-1,"/search_exp.php","bsn");
   client_state csp;
   http_response rsp;

src/plugins/httpserv/httpserv.cpp

@@ -1110,6 +1110,8 @@ t.dtd\"><html><head><title>408 - Seeks fail connection to background search engi
 
   void httpserv::find_bqc(struct evhttp_request *r, void *arg)
   {
+    errlog::log_error(LOG_LEVEL_DEBUG, "httpserv::find_bqc(): CALLED!");
+
     client_state csp;
     csp._config = seeks_proxy::_config;
     http_response *rsp = new http_response();
@@ -1171,6 +1173,7 @@ t.dtd\"><html><head><title>408 - Seeks fail connection to background search engi
                                            input_buffer->off / sizeof(u_char));
 #endif
 
+    errlog::log_error(LOG_LEVEL_DEBUG, "httpserv::find_bqc(): post_content=%s,length()=%d", post_content.c_str(), post_content.length());
     if (post_content.empty())
       {
         httpserv::reply_with_error_400(r); //TODO: proper error type.
@@ -1191,6 +1194,7 @@ t.dtd\"><html><head><title>408 - Seeks fail connection to background search engi
       miscutil::enlist_unique_header(&csp._headers,"host",host);
 
     // call to find_bqc callback.
+    errlog::log_error(LOG_LEVEL_DEBUG, "httpserv::find_bqc(): csp._iob._cur=%s", csp._iob._cur);
     sp_err serr = udb_service::cgi_find_bqc(&csp,rsp,parameters);
     miscutil::list_remove_all(&csp._headers);
     delete[] csp._iob._cur;

src/plugins/query_capture/db_query_record_msg.proto

@@ -33,5 +33,5 @@ message related_query
 
 extend sp.db.record
 {
- required related_queries queries = 20; /* original queries */
+ optional related_queries queries = 20; /* original queries */
 }

src/plugins/query_capture/query_capture.cpp

@@ -650,7 +650,7 @@ namespace seeks_plugins
                 referer = "";
               }
           }
-        else if (miscutil::strncmpic((*lit),"Seeks-Remote-Location:",22) == 0)
+        else if (miscutil::strncmpic((*lit),"-Seeks-Remote-Location:",22) == 0)
           {
             base_url = (*lit);
             try

src/plugins/udb_service/halo_msg_wrapper.cpp

@@ -29,6 +29,8 @@ namespace seeks_plugins
                                      uint32_t &expansion,
                                      std::vector<std::string> &hashes) throw (sp_exception)
   {
+    errlog::log_error(LOG_LEVEL_DEBUG, "halo_msg_wrapper::deserialize(): msg=%s,expansion=%d -  CALLED!", msg.c_str(), expansion);
+
     hash_halo h;
     if (!h.ParseFromString(msg))
       {
@@ -39,14 +41,19 @@ namespace seeks_plugins
     expansion = h.expansion();
     for (int i=0; i<h.key_size(); i++)
       hashes.push_back(h.key(i));
+
+    errlog::log_error(LOG_LEVEL_DEBUG, "halo_msg_wrapper::deserialize(): msg=%s,expansion=%d -  EXIT!", msg.c_str(), expansion);
   }
 
   void halo_msg_wrapper::serialize(const uint32_t &expansion,
                                    const hash_multimap<uint32_t,DHTKey,id_hash_uint> &qhashes,
                                    std::string &msg) throw (sp_exception)
   {
+    errlog::log_error(LOG_LEVEL_DEBUG, "halo_msg_wrapper::serialize(): msg=%s,expansion=%d - CALLED!", msg.c_str(), expansion);
+
     hash_halo h;
     h.set_expansion(expansion);
+    errlog::log_error(LOG_LEVEL_DEBUG, "halo_msg_wrapper::serialize(): h.expansion()=%d", h.expansion());
     hash_multimap<uint32_t,DHTKey,id_hash_uint>::const_iterator hit = qhashes.begin();
     while(hit!=qhashes.end())
       {
@@ -60,6 +67,7 @@ namespace seeks_plugins
         errlog::log_error(LOG_LEVEL_ERROR,msg.c_str());
         throw sp_exception(UDBS_ERR_SERIALIZE,msg);
       }
-  }
 
+    errlog::log_error(LOG_LEVEL_DEBUG, "halo_msg_wrapper::serialize(): msg=%s,expansion=%d - EXIT!", msg.c_str(), expansion);
+  }
 } /* end of namespace. */

src/plugins/udb_service/udb_client.cpp

@@ -118,6 +118,7 @@ namespace seeks_plugins
     std::string msg;
     try
       {
+        errlog::log_error(LOG_LEVEL_DEBUG, "udb_client::find_bqc(): Calling halo_msg_wrapper::serialize(%d,???,%s) ...", expansion, msg.c_str());
         halo_msg_wrapper::serialize(expansion,qhashes,msg);
       }
     catch(sp_exception &e)
@@ -135,7 +136,7 @@ namespace seeks_plugins
     std::vector<std::string> urls;
     urls.reserve(1);
     urls.push_back(url);
-    errlog::log_error(LOG_LEVEL_DEBUG,"call: %s",url.c_str());
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_client::find_bqc(): url=%s,msg=%s,msg.length()=%d", url.c_str(), msg.c_str(), msg.length());
     std::vector<int> status;
     if (udb_service_configuration::_config->_p2p_proxy_addr.empty())
       cmg.www_mget(urls,1,NULL,"",0,status,
@@ -145,6 +146,7 @@ namespace seeks_plugins
                         udb_service_configuration::_config->_p2p_proxy_addr,
                         udb_service_configuration::_config->_p2p_proxy_port,
                         status,NULL,NULL,"POST",&msg,msg.length()*sizeof(char));
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_client::find_bqc(): status[0]=%d", status[0]);
     if (status[0] !=0)
       {
         // failed connection.

src/plugins/udb_service/udb_server.cpp

@@ -21,6 +21,7 @@
 #include "db_query_record.h"
 #include "cf.h"
 #include "seeks_proxy.h"
+#include "encode.h"
 #include "errlog.h"
 
 #include <string.h>
@@ -67,6 +68,8 @@ namespace seeks_plugins
 
     uint32_t expansion = 0;
     std::vector<std::string> qhashes;
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_server::find_bqc_cb(): content=%s", content.c_str());
+
     try
       {
         halo_msg_wrapper::deserialize(content,
@@ -94,12 +97,14 @@ namespace seeks_plugins
       dbr->print(std::cerr);*/
 
     // fill up response.
-    size_t body_size = str.length() * sizeof(char);
+    std::string* encoded = new std::string(encode::url_encode(str.c_str()));
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_server::find_bqc_cb(): encoded=%s(%d)", encoded->c_str(), encoded->length());
+    size_t body_size = encoded->length() * sizeof(char);
     if (!rsp->_body)
       rsp->_body = (char*)std::malloc(body_size);
     rsp->_content_length = body_size;
-    for (size_t i=0; i<str.length(); i++)
-      rsp->_body[i] = str[i];
+    for (size_t i=0; i<encoded->length(); i++)
+      rsp->_body[i] = encoded->at(i);
     delete dbr;
     return SP_ERR_OK;
   }

src/plugins/udb_service/udb_service.cpp

@@ -21,7 +21,9 @@
 #include "udb_client.h"
 #include "udb_service_configuration.h"
 #include "seeks_proxy.h"
+#include "encode.h"
 #include "miscutil.h"
+#include "errlog.h"
 
 #include <string.h>
 #include <sys/stat.h>
@@ -97,11 +99,21 @@ namespace seeks_plugins
                                    http_response *rsp,
                                    const hash_map<const char*,const char*,hash<const char*>,eqstr> *parameters)
   {
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_service::cgi_find_bqc(): CALLED!");
     if (!seeks_proxy::_user_db)
       {
         return SP_ERR_FILE; // no user db.
       }
-    std::string content = std::string(csp->_iob._cur);//,csp->_iob._size); // XXX: beware...
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_service::cgi_find_bqc(): Calling udb_server::find_bqc_cb(): _buf=%s", csp->_iob._buf);
+    char *decoded = encode::url_decode(csp->_iob._cur);
+    if (!decoded)
+      {
+        // Decoding didn't work
+        errlog::log_error(LOG_LEVEL_ERROR, "udb_service::cgi_find_bqc(): Decoding didn't work.");
+        return SP_ERR_CGI_PARAMS;
+      }
+    std::string content = std::string(decoded);//,csp->_iob._size); // XXX: beware...
+    errlog::log_error(LOG_LEVEL_DEBUG, "udb_service::cgi_find_bqc(): Calling udb_server::find_bqc_cb(): content=%s,length()=%d,_size=%d,%E", content.c_str(), content.length(), csp->_iob._size, sizeof(csp->_iob));
     return udb_server::find_bqc_cb(content,rsp);
   }
 

src/plugins/websearch/query_context.cpp

@@ -590,6 +590,8 @@ namespace seeks_plugins
 
   std::string query_context::detect_base_url_http(client_state *csp)
   {
+    errlog::log_error(LOG_LEVEL_DEBUG, "query_context::detect_base_url_http(): CALLED!");
+
     std::list<const char*> headers = csp->_headers;
 
     // first we try to get base_url from a custom header
@@ -604,6 +606,7 @@ namespace seeks_plugins
             try
               {
                 base_url = base_url.substr(pos+1);
+                errlog::log_error(LOG_LEVEL_DEBUG, "query_context::detect_base_url_http(): base_url=%s found in headers", base_url.c_str());
               }
             catch (std::exception &e)
               {
@@ -638,7 +641,10 @@ namespace seeks_plugins
             ++sit;
           }
         base_url = csp->_http._ssl ? "https://" : "http://" + base_url;
+        errlog::log_error(LOG_LEVEL_DEBUG, "query_context::detect_base_url_http(): base_url=%s with no custom header", base_url.c_str());
       }
+
+    errlog::log_error(LOG_LEVEL_DEBUG, "query_context::detect_base_url_http(): base_url=%s - EXIT!", base_url.c_str());
     return base_url;
   }