/** * @file apps/fetch.c * @brief Obtain files over HTTP. * * @copyright * This file is part of ToaruOS and is released under the terms * of the NCSA / University of Illinois License - see LICENSE.md * Copyright (C) 2015-2021 K. Lange */ #include #include #include #include #include #include #include #include #include #include #include #include #include #define SIZE 512 #define BOUNDARY "------ToaruOSFetchUploadBoundary" struct http_req { char domain[SIZE]; char path[SIZE]; int port; int ssl; }; struct { int show_headers; const char * output_file; const char * cookie; FILE * out; int prompt_password; const char * upload_file; char * password; int show_progress; size_t content_length; size_t size; struct timeval start; int calculate_output; int slow_upload; int machine_readable; } fetch_options = {0}; int parse_url(char * d, struct http_req * r) { if (strstr(d, "http://") == d) { d += strlen("http://"); r->port = 80; r->ssl = 0; } else if (strstr(d, "https://") == d) { d += strlen("https://"); r->port = 443; r->ssl = 1; } else { fprintf(stderr, "Unrecognized protocol: %s\n", d); return 1; } char * s = strstr(d, "/"); if (!s) { strcpy(r->domain, d); strcpy(r->path, ""); } else { *s = 0; s++; strcpy(r->domain, d); strcpy(r->path, s); } if (strstr(r->domain,":")) { char * port = strstr(r->domain,":"); *port = '\0'; port++; r->port = atoi(port); } return 0; } #define BAR_WIDTH 20 #define bar_perc "||||||||||||||||||||" #define bar_spac " " void print_progress(int force) { static uint64_t last_size = 0; if (!fetch_options.show_progress) return; if (!force && (last_size + 102400 > fetch_options.size)) return; last_size = fetch_options.size; struct timeval now; gettimeofday(&now, NULL); fprintf(stderr,"\033[?25l\033[G%6dkB",(int)fetch_options.size/1024); if (fetch_options.content_length) { int percent = (fetch_options.size * BAR_WIDTH) / (fetch_options.content_length); fprintf(stderr," / %6dkB [%.*s%.*s]", (int)fetch_options.content_length/1024, percent,bar_perc,BAR_WIDTH-percent,bar_spac); } double timediff = (double)(now.tv_sec - fetch_options.start.tv_sec) + (double)(now.tv_usec - fetch_options.start.tv_usec)/1000000.0; if (timediff > 0.0) { double rate = (double)(fetch_options.size) / timediff; double s = rate/(1024.0) * 8.0; if (s > 1024.0) { fprintf(stderr," %.2f Mbps", s/1024.0); } else { fprintf(stderr," %.2f Kbps", s); } if (!force && fetch_options.content_length) { if (rate > 0.0) { double remaining = (double)(fetch_options.content_length - fetch_options.size) / rate; fprintf(stderr," (%.2f sec remaining)", remaining); } } else { fprintf(stderr," (%.2f sec elapsed)", timediff); } } fprintf(stderr,"\033[K\033[?25h"); fflush(stderr); } int usage(char * argv[]) { fprintf(stderr, "fetch - download files over HTTP\n" "\n" "usage: %s [-hOvmp?] [-c cookie] [-o file] [-u file] [-s speed] URL\n" "\n" " -h \033[3mshow headers\033[0m\n" " -O \033[3msave the file based on the filename in the URL\033[0m\n" " -v \033[3mshow progress\033[0m\n" " -m \033[3mmachine readable output\033[0m\n" " -p \033[3mprompt for password\033[0m\n" " -c ... \033[3mset cookies\033[0m\n" " -o ... \033[3msave to the specified file\033[0m\n" " -u ... \033[3mupload the specified file\033[0m\n" " -s ... \033[3mspecify the speed for uploading slowly\033[0m\n" " -? \033[3mshow this help text\033[0m\n" "\n", argv[0]); return 1; } int collect_password(char * password) { fprintf(stdout, "Password for upload: "); fflush(stdout); /* Disable echo */ struct termios old, new; tcgetattr(fileno(stdin), &old); new = old; new.c_lflag &= (~ECHO); tcsetattr(fileno(stdin), TCSAFLUSH, &new); fgets(password, 1024, stdin); password[strlen(password)-1] = '\0'; tcsetattr(fileno(stdin), TCSAFLUSH, &old); fprintf(stdout, "\n"); return 0; } #define MAX_HTTP_LINE 1024 void read_http_line(char * buf, FILE * f) { memset(buf, 0x00, MAX_HTTP_LINE); fgets(buf, MAX_HTTP_LINE-1, f); char * _r = strchr(buf, '\r'); if (_r) { *_r = '\0'; } if (!_r) { _r = strchr(buf, '\n'); /* that's not right, but, whatever */ if (_r) { *_r = '\0'; } } } void bad_response(void) { fprintf(stderr, "Bad response.\n"); exit(1); } int http_fetch(FILE * f) { hashmap_t * headers = hashmap_create(10); /* Parse response */ { char buf[MAX_HTTP_LINE]; read_http_line(buf, f); char * elements[3]; elements[0] = buf; elements[1] = strchr(elements[0], ' '); if (!elements[1]) bad_response(); *elements[1] = '\0'; elements[1]++; elements[2] = strchr(elements[1], ' '); if (!elements[2]) bad_response(); *elements[2] = '\0'; elements[2]++; if (strcmp(elements[1], "200")) { fprintf(stderr, "Bad response code: %s\n", elements[1]); return 1; } } /* Parse headers */ while (1) { char buf[MAX_HTTP_LINE]; read_http_line(buf, f); if (!*buf) { break; } /* Split */ char * name = buf; char * value = strstr(buf, ": "); if (!value) bad_response(); *value = '\0'; value += 2; hashmap_set(headers, name, strdup(value)); } if (fetch_options.show_headers) { list_t * hash_keys = hashmap_keys(headers); foreach(_key, hash_keys) { char * key = (char *)_key->value; fprintf(stderr, "[%s] = %s\n", key, (char*)hashmap_get(headers, key)); } list_free(hash_keys); free(hash_keys); } /* determine how many bytes we should read now */ if (!hashmap_has(headers, "Content-Length")) { fprintf(stderr, "Don't know how much to read.\n"); return 1; } int bytes_to_read = atoi(hashmap_get(headers, "Content-Length")); fetch_options.content_length = bytes_to_read; gettimeofday(&fetch_options.start, NULL); while (bytes_to_read > 0) { char buf[1024]; size_t r = fread(buf, 1, bytes_to_read < 1024 ? bytes_to_read : 1024, f); fwrite(buf, 1, r, fetch_options.out); fetch_options.size += r; print_progress(0); if (fetch_options.machine_readable && fetch_options.content_length) { fprintf(stdout,"%zu %zu\n",fetch_options.size,fetch_options.content_length); } bytes_to_read -= r; } print_progress(1); return 0; } int main(int argc, char * argv[]) { int opt; while ((opt = getopt(argc, argv, "?c:hmo:Opu:vs:")) != -1) { switch (opt) { case '?': return usage(argv); case 'O': fetch_options.calculate_output = 1; break; case 'c': fetch_options.cookie = optarg; break; case 'h': fetch_options.show_headers = 1; break; case 'o': fetch_options.output_file = optarg; break; case 'u': fetch_options.upload_file = optarg; break; case 'v': fetch_options.show_progress = 1; break; case 'm': fetch_options.machine_readable = 1; break; case 'p': fetch_options.prompt_password = 1; break; case 's': fetch_options.slow_upload = atoi(optarg); break; } } if (optind >= argc) { return usage(argv); } struct http_req my_req; if (parse_url(argv[optind], &my_req)) { return 1; } if (my_req.ssl) { /* TODO look for a viable backend. */ fprintf(stderr, "%s: no tls backend\n", argv[0]); return 1; } if (fetch_options.calculate_output) { char * tmp = strdup(my_req.path); char * x = strrchr(tmp,'/'); if (x) { tmp = x + 1; } fetch_options.output_file = tmp; } fetch_options.out = stdout; if (fetch_options.output_file) { fetch_options.out = fopen(fetch_options.output_file, "w+"); } int sock = socket(AF_INET, SOCK_STREAM, 0); if (sock < 0) { perror("socket"); return 1; } struct hostent * remote = gethostbyname(my_req.domain); if (!remote) { perror("gethostbyname"); return 1; } struct sockaddr_in addr; addr.sin_family = AF_INET; memcpy(&addr.sin_addr.s_addr, remote->h_addr, remote->h_length); addr.sin_port = htons(my_req.port); if (connect(sock, (struct sockaddr*)&addr, sizeof(struct sockaddr_in)) < 0) { perror("connect"); return 1; } FILE * f = fdopen(sock,"w+"); if (!f) { fprintf(stderr, "Nope.\n"); return 1; } if (fetch_options.prompt_password) { fetch_options.password = malloc(100); collect_password(fetch_options.password); } if (fetch_options.upload_file) { FILE * in_file = fopen(fetch_options.upload_file, "r"); srand(time(NULL)); int boundary_fuzz = rand(); char tmp[512]; size_t out_size = 0; if (fetch_options.password) { out_size += sprintf(tmp, "--" BOUNDARY "%08x\r\n" "Content-Disposition: form-data; name=\"password\"\r\n" "\r\n" "%s\r\n",boundary_fuzz, fetch_options.password); } out_size += strlen("--" BOUNDARY "00000000\r\n" "Content-Disposition: form-data; name=\"file\"; filename=\"\"\r\n" "Content-Type: application/octet-stream\r\n" "\r\n" /* Data goes here */ "\r\n" "--" BOUNDARY "00000000" "--\r\n"); out_size += strlen(fetch_options.upload_file); fseek(in_file, 0, SEEK_END); out_size += ftell(in_file); fseek(in_file, 0, SEEK_SET); fprintf(f, "POST /%s HTTP/1.0\r\n" "User-Agent: curl/7.35.0\r\n" "Host: %s\r\n" "Accept: */*\r\n" "Content-Length: %d\r\n" "Content-Type: multipart/form-data; boundary=" BOUNDARY "%08x\r\n" "\r\n", my_req.path, my_req.domain, (int)out_size, boundary_fuzz); fprintf(f,"%s",tmp); fprintf(f, "--" BOUNDARY "%08x\r\n" "Content-Disposition: form-data; name=\"file\"; filename=\"%s\"\r\n" "Content-Type: application/octet-stream\r\n" "\r\n", boundary_fuzz, fetch_options.upload_file); while (!feof(in_file)) { char buf[1024]; size_t r = fread(buf, 1, 1024, in_file); fwrite(buf, 1, r, f); if (fetch_options.slow_upload) { usleep(1000 * fetch_options.slow_upload); /* TODO fix terrible network stack; hopefully this ensures we send stuff right. */ } } fclose(in_file); fprintf(f,"\r\n--" BOUNDARY "%08x--\r\n", boundary_fuzz); fflush(f); } else if (fetch_options.cookie) { fprintf(f, "GET /%s HTTP/1.0\r\n" "User-Agent: curl/7.35.0\r\n" "Host: %s\r\n" "Accept: */*\r\n" "Cookie: %s\r\n" "\r\n", my_req.path, my_req.domain, fetch_options.cookie); } else { fprintf(f, "GET /%s HTTP/1.0\r\n" "User-Agent: curl/7.35.0\r\n" "Host: %s\r\n" "Accept: */*\r\n" "\r\n", my_req.path, my_req.domain); } http_fetch(f); fflush(fetch_options.out); if (fetch_options.show_progress) { fprintf(stderr,"\n"); } if (fetch_options.machine_readable) { fprintf(stdout,"done\n"); } return 0; }