/* * Copyright (C) 2010 Robin H.Johnson, Ovechko Kostyantyn . * * Project: IDFetch. * Developer: Ovechko Kostyantyn Olexandrovich (Kharkiv State Technical University of Construction and Architecture, Ukraine). * Mentor: Robin H. Johnson (Gentoo Linux: Developer, Trustee & Infrastructure Lead). * Mentoring organization: Gentoo Linux. * Sponsored by GSOC 2010. * * This file is part of Segget. * * Segget is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * Segget is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with Segget; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "connection.h" #include "utils.h" #include "settings.h" long script_waiting_connection_num=-1; uint Tconnection::total_connections=0; Tconnection connection_array[MAX_CONNECTS]; void init_connections(){ try{ for (ulong connection_num=0; connection_numparent_distfile->name); mirror_num=best_mirror_num; network_num=network_number; total_dld_bytes=0; bytes_per_last_interval=0; gettimeofday(&start_time,NULL); debug("Connecting network"+toString(network_num)); segment->parent_distfile->set_status(DDOWNLOADING); if (network_array[network_num].network_mode==MODE_PROXY_FETCHER){ connection_start_time_network_phase_for_pf_networks=segment->parent_distfile->network_distfile_brokers_array[network_num].phase; } Tmirror *Pcurr_mirror; switch (network_array[network_num].network_mode){ case MODE_REMOTE:{ url=segment->parent_distfile->url_list[mirror_num]; Pcurr_mirror=find_mirror(strip_mirror_name(url)); break; } case MODE_CORAL_CDN:{ url=convert_to_coral_cdn_url(segment->parent_distfile->url_list[mirror_num]); Pcurr_mirror=find_mirror(strip_mirror_name(url)); break; } default:{ Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; url=Pcurr_mirror->url+segment->parent_distfile->name; } } debug(" URL:"+url); if (run_user_python_script(connection_num)){ if (segment->parent_distfile->active_connections_num<=0){ segment->parent_distfile->set_status(DSCRIPTREJECTED); } return REJECTED_BY_USER_PYTHON_SCRIPT; } stats.active_connections_counter++; ulong segment_start_result=segment->start(cm, connection_num, network_num, distfile_num, url); if (segment_start_result){ return segment_start_result; }else{ segment->parent_distfile->active_connections_num++; active=true; Pcurr_mirror->start(); network_array[network_num].connect(); } debug("Started connection for distfile: "+segment->parent_distfile->name); return 0; }catch(...){ error_log("Error in connection.cpp: start()"); } return ERROR_WHILE_PREPARING_CONNECTION; } void Tconnection::stop(CURLcode connection_result){ try{ stats.active_connections_counter--; Tmirror *Pcurr_mirror; switch (network_array[network_num].network_mode){ case MODE_REMOTE: case MODE_CORAL_CDN:{ Pcurr_mirror=find_mirror(strip_mirror_name(url)); break; } default:{ Pcurr_mirror=&network_array[network_num].benchmarked_mirror_list[mirror_num]; } } debug("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); if (connection_result){ string error_str=curl_easy_strerror(connection_result); debug(" ERROR "+toString(connection_result)+": "+error_str); error_log("Finished connection for distfile: "+segment->parent_distfile->name+" Segment#:"+toString(segment->segment_num)+" Network#"+toString(network_num)+" Status: "+toString(connection_result)); error_log(" ERROR "+toString(connection_result)+": "+error_str); } active=false; network_array[network_num].disconnect(); // network_array[network_num].benchmarked_mirror_list[mirror_num].stop(); segment->segment_file.close(); if (connection_result==0){ if (! segment->segment_verification_is_ok()){ connection_result=CURLE_READ_ERROR; Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); debug("libcurl stated succesful result but size for the segment is incorrect"); error_log("libcurl stated succesful result but size for the segment is incorrect"); }else{ Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),segment->segment_size); } }else{ Pcurr_mirror->stop(time_left_since(connection_array[connection_num].start_time),0); } segment->parent_distfile->active_connections_num--; msg_clean_connection(connection_num); debug("before gettimeofday"); timeval now_time; gettimeofday(&now_time,NULL); if (connection_result!=0){ stats.fails_counter++; switch (network_array[network_num].network_mode){ case MODE_LOCAL:{ // prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; debug("before setting mirror fail"); segment->parent_distfile->network_distfile_brokers_array[network_num].local_mirror_failed(mirror_num); debug("after setting mirror fail"); // find_mirror(strip_mirror_name(segment->url)); break; } case MODE_PROXY_FETCHER:{ // prnt_distfile->network_distfile_brokers_array[network_num].mirror_fails_vector[mirror_num]=true; if (connection_start_time_network_phase_for_pf_networks==E_USE_AS_LOCAL_MIRRORS){ segment->parent_distfile->network_distfile_brokers_array[network_num].local_mirror_failed(mirror_num); }else{ // proxy-fetcher mirror failed, if everything correct it must be in phase E_PROXY_FETCHER_DOWNLOADED, segment->parent_distfile->network_distfile_brokers_array[network_num].proxy_fetcher_mirror_failed(mirror_num); } // find_mirror(strip_mirror_name(segment->url)); break; } default:{ // MODE REMOTE MIRRORS } } // error -> start downloading again // msg_status2(segment->connection_num, toString(connection_result)+"]- Failed download "+segment->file_name); debug(toString(connection_result)+"]- Failed download "+segment->url); if (segment->try_num>=settings.max_tries){ segment->status=SFAILED; segment->parent_distfile->set_status(DFAILED); error_log("Segget failed to download distfile: "+segment->parent_distfile->name); error_log("Segment:"+segment->file_name+" has reached max_tries limit - segment.status set to FAILED"); } else segment->status=SWAITING; }else{ // no error => count this one and start new log("Succesfully downloaded "+segment->file_name+" on connection#"+toString(connection_num)); debug(" Successful download "+segment->url); segment->status=SDOWNLOADED; segment->parent_distfile->inc_dld_segments_count(segment); }; if ((segment->parent_distfile->get_status()!=DFAILED) and (segment->parent_distfile->get_status()!=DDOWNLOADED) and (segment->parent_distfile->get_status()!=DALL_LM_AND_PF_MIRRORS_FAILED)){ if (segment->parent_distfile->active_connections_num>0){ segment->parent_distfile->set_status(DDOWNLOADING); }else{ segment->parent_distfile->set_status(DWAITING); } } }catch(...){ error_log("Error in connection.cpp: stop()"); } } void Tconnection::inc_bytes_per_last_interval(ulong new_bytes_count){ try{ total_dld_bytes+=new_bytes_count; bytes_per_last_interval+=new_bytes_count; }catch(...){ error_log("Error in connection.cpp: inc_bytes_per_last_interval()"); } } void Tconnection::show_connection_progress(ulong time_diff){ try{ stats.total_bytes_per_last_interval+=bytes_per_last_interval; ulong speed=(bytes_per_last_interval*1000)/time_diff; ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); string eta_string; if (avg_speed==0){ eta_string=" ETA: inf"; }else{ eta_string=" ETA: "+secsToString((segment->segment_size-segment->downloaded_bytes)/avg_speed); } string speed_str; string avg_speed_str; speed_str=" Speed: "+speedToString(speed); avg_speed_str=" AVG speed: "+speedToString(avg_speed); int percent=segment->downloaded_bytes*100/segment->segment_size; string network_type_str; switch (network_array[network_num].network_mode){ case MODE_REMOTE: network_type_str="REM"; break; case MODE_PROXY_FETCHER: network_type_str="P-F"; break; case MODE_LOCAL: network_type_str="LOC"; break; case MODE_CORAL_CDN: network_type_str="CDN"; break; } string progress_text=field("[Net",network_num,1) +":"+network_type_str+"]" +field(" Segment:",segment->segment_num, 5) +field(" Try:",segment->try_num,4) +field(" Bytes:",segment->downloaded_bytes,7) +field(" / ",segment->segment_size,7) +field(" = ",percent,3)+"%" +speed_str +avg_speed_str +eta_string; msg_segment_progress(segment->connection_num, progress_text); bytes_per_last_interval=0; }catch(...){ error_log("Error in connection.cpp: show_connection_progress()"); } } string Tconnection::get_html_connection_progress(){ try{ ulong time_diff=time_left_since(stats.previous_time); // stats.total_bytes_per_last_interval+=bytes_per_last_interval; ulong speed=(bytes_per_last_interval*1000)/time_diff; ulong avg_speed=(total_dld_bytes*1000)/time_left_since(start_time); string eta_string; if (avg_speed==0){ eta_string="inf"; }else{ eta_string=secsToString((segment->segment_size-segment->downloaded_bytes)/avg_speed); } string speed_str; string avg_speed_str; speed_str=speedToString(speed); avg_speed_str=speedToString(avg_speed); int segment_percent; if (segment->segment_size>0){ segment_percent=segment->downloaded_bytes*100/segment->segment_size; }else{ segment_percent=100; } int distfile_percent; if (segment->parent_distfile->size>0){ distfile_percent=segment->parent_distfile->dld_bytes*100/segment->parent_distfile->size; }else{ distfile_percent=100; } int unfinished_segments_distfile_percent; //TO-DO: it's necessary to check all connections and add dld bytes to be correct !!! if (segment->parent_distfile->size>0){ unfinished_segments_distfile_percent=segment->downloaded_bytes*100/segment->parent_distfile->size; }else{ unfinished_segments_distfile_percent=0; } string network_type_str; switch (network_array[network_num].network_mode){ case MODE_REMOTE: network_type_str="Remote"; break; case MODE_PROXY_FETCHER: network_type_str="Proxy-Fetcher"; break; case MODE_LOCAL: network_type_str="Local"; break; case MODE_CORAL_CDN: network_type_str="CDN"; break; } string progress_text= (string)"\n\n\n" +((distfile_percent>0)?"\n" :"") +((unfinished_segments_distfile_percent>0)?"\n" :"") +((100-distfile_percent-unfinished_segments_distfile_percent>0)?"\n" :"") +"\n
\n\n\n\n\n\n
\n" // +" "+toString(distfile_percent+unfinished_segments_distfile_percent)+"%" +toString(distfile_percent)+"%" +"\n\n" +"\n" +"\n\n\n" +"\n\n\n" +"
"+segment->parent_distfile->name+"
\n"+segment->url+"
\n" +"" +((segment_percent>0)?"" :"") +((100-segment_percent>0)?"" :"") +"
" +toString(segment_percent)+"%" +""+toString(segment->segment_num) +""+toString(segment->try_num) +""+toString(network_num) +""+network_type_str +""+toString(segment->downloaded_bytes) +""+toString(segment->segment_size) +""+speed_str +""+avg_speed_str +""+eta_string +""; return progress_text; }catch(...){ error_log("Error in connection.cpp: show_connection_progress()"); } return ""; }