diff --git a/docs/man/ytfzf.1 b/docs/man/ytfzf.1 index eabaa5f0..0f51e213 100644 --- a/docs/man/ytfzf.1 +++ b/docs/man/ytfzf.1 @@ -200,7 +200,7 @@ Scrapers .TP .BI \-c " scrapers" ", " "\-\-scrape=scrapers" Set which scraper to use. -Multiple scrapers can be seperated by , +Multiple scrapers can be separated by , The currently supported builtin scrapers are: .RS .TP @@ -270,7 +270,7 @@ convert the invidious links to youtube links before playing/downloading. Scraper Options .RS .PP -Currently, \-\-sort\-by, \-\-upload\-date, \-\-video\-duration, \-\-type, \-\-thumbnail\-quality, and \-\-features only applies to the scrape: youtube/Y +Currently, \-\-video\-duration, \-\-type, \-\-thumbnail\-quality, and \-\-features only applies to the scrape: youtube/Y .TP .BR \-D ", " \-\-external\-menu Use an external menu instead of fzf. @@ -288,21 +288,38 @@ Amount of videos to scrape on odysee. This can also be set in the config file with .BR odysee_video_search_count . .TP +.BR "\-\-nsfw" +Whether or not to search for nsfw videos. +.br +Only works with odysee/O +This can also be set in the config file with +.BR nsfw . +.TP .BI "\-\-sort\-by=sort" +Works with youtube/Y and odysee/O. +.br +To use a different sort for each scrape, use a "," to separate the sorts. +.br As apposed to \-\-sort, this happens during the serach, not after. Results should sort by: .RS .TP .IR relevance .TP -.IR rating +.IR rating " (youtube only)" .TP .IR upload_date .TP -.IR view_count +.IR oldest_first " (odysee only)" +.TP +.IR view_count " (youtube only)" .RE .TP .BI "\-\-upload\-date=time\-frame" +Works with, youtube/Y and odysee/O +.br +To use a different sort for each scrape, use a "," to separate the dates. +.br Search for videos within the last: .RS .TP @@ -368,7 +385,7 @@ The end frame of the video (low quality) .RE .TP .BI "\-\-features=features" -The features to have on a video (comma seperated). +The features to have on a video (comma separated). .RS .TP .IR hd diff --git a/docs/man/ytfzf.5 b/docs/man/ytfzf.5 index 8a3ca5c8..80ec5c17 100644 --- a/docs/man/ytfzf.5 +++ b/docs/man/ytfzf.5 @@ -422,11 +422,13 @@ The attribute to sort by when searching. .TP .IR relevance " (default)" .TP -.IR rating +.IR rating " (youtube only)" .TP .IR upload_date .TP -.IR view_count +.IR oldest_first " (odysee only)" +.TP +.IR view_count " (youtbe only)" .RE .TP @@ -470,6 +472,12 @@ The type of results to get. .IR all " (may not work on some instances)" .RE +.TP +.RB $ nsfw +Whether or not to search for nsfw videos in odysee/O. +.br +.RI default: " false" + .TP .RB $ search_result_features The features to have on a video (comma seperated). @@ -873,6 +881,16 @@ This function takes 1 argument: The action. .EE +.TP +.BR manage_multi_custom_filters () +This function should add the ability for filters to be used in multiple scrapes, +.br +To see how this is done, take a look at the manage_multi_filters() function in ytfzf. +.br +This function is called before the website is scraped. +.br +This function takes no arguments. + .RE .SH VIDEO JSON FORMAT @@ -922,9 +940,18 @@ To easily get the formated json for a video, run .SH CUSTOM SCRAPERS .PP -Custom scrapers are programs located in $YTFZF_CUSTOM_SCRAPERS_DIR. +Custom scrapers shell scripts located in $YTFZF_CUSTOM_SCRAPERS_DIR. +.br +A scraper is responsible for scraping videos from a website and APPENDING them to "$ytfzf_video_json_file". +.br +The shell script must be the same shell as your /bin/sh. .br -To customize some behavior of the scraper see CUSTOM SCRAPER CONFIG +In addition, the script must also define the function +.I interface_ +.br +With _ replacing \-. +.br +This function could handle everything itself, or call another program written in any language to handle it. .RE .PP A custom scraper will take the search query as the first argument to the program @@ -936,70 +963,27 @@ The JSON should be structured as described in VIDEO JSON FORMAT The final JSON shall be .B APPENDED to the file given as an argument. -.PP -A custom scraper can be written in any programming language so long as the file can be run as a normal command .RE -.SH CUSTOM SCRAPER CONFIG +.SH CUSTOM INTERFACES .PP -To customize some behavior of a scraper create a file named .conf. (replace with the name of the scraper) +Custom interfaces are shell scripts located in $YTFZF_CUSTOM_INTERFACES_DIR. .br -This file would also be located in $YTFZF_CUSTOM_SCRAPER_DIR. -.PP -The syntax of this file is as such: -.RS -.EX -key: value -key2: value2 -.EE -.RE -There are no quotations around anything. - -.SS CUSTOM SCRAPER CONFIG OPTIONS -.PP -Options to use as keys in .conf -.TP -.RB vars -The variables to pass into the scraper after the search and json file path. +An interface is responsible for letting the user pick a video from "$ytfzf_video_json_file", then writing the url(s) to "$ytfzf_selected_urls" .br -Example: -.RS -.EX -vars: $is_auto_select $is_sort -.EE -This will make the 3rd argument the value of $is_auto_select, and the 4th $is_sort -.RE - -.SH CUSTOM INTERFACES -.PP -Custom interfaces are programs located in $YTFZF_CUSTOM_INTERFACES_DIR. +The shell script must be the same shell as your /bin/sh. .br -To customize some behavior of the interface see CUSTOM INTERFACE CONFIG -.RE -.PP -A custom interface will take a path to the json file holding all data about all the videos as the first argument. +In addition, the script must also define the function +.I interface_ .br -The second argument will be a path to a file to store the selected url in, separated by new lines. - -.SH CUSTOM INTERFACE CONFIG -.PP -Similar to CUSTOM SCRAPERS, some behavior of an interface can be changed in .conf +With _ replacing \-. .br -This file should also be in $YTFZF_CUSTOM_INTERFACES_DIR. +This function could handle everything itself, or call another program written in any language to handle it. +.RE .PP -The syntax is the same as CUSTOM SCRAPER CONFIG - -.SS CUSTOM INTERFACE CONFIG OPTIONS -.TP -.RB vars -The variabels to pass into the interface after the 2 file paths. +interface_ will take a path to the json file holding all data about all the videos as the first argument. .br -Example: -.RS -.EX -vars: $show_thumbnails $is_sort -.EE -This will make the 3rd argument the value of $show_thumbnails, and the 4th, $is_sort +The second argument will be a path to a file to store the selected url in, separated by new lines. .SH THUMBNAIL VIEWERS Custom thumbnail viewers are programs in $YTFZF_THUMBNAIL_VIEWERS_DIR. diff --git a/ytfzf b/ytfzf index ccfd707c..55660ef1 100755 --- a/ytfzf +++ b/ytfzf @@ -136,7 +136,7 @@ dep_check "jq" || die 3 "jq is a required dependancy, please install it\n" : "${YTFZF_CONFIG_FILE:=$YTFZF_CONFIG_DIR/conf.sh}" : "${YTFZF_SUBSCRIPTIONS_FILE:=$YTFZF_CONFIG_DIR/subscriptions}" : "${YTFZF_THUMBNAIL_VIEWERS_DIR:=$YTFZF_CONFIG_DIR/thumbnail-viewers}" -: "${YTFZF_CUSTOM_INTERFACE_DIR:=$YTFZF_CONFIG_DIR/interfaces}" +: "${YTFZF_CUSTOM_INTERFACES_DIR:=$YTFZF_CONFIG_DIR/interfaces}" [ -f "$YTFZF_CONFIG_FILE" ] && . "$YTFZF_CONFIG_FILE" #}}} @@ -276,6 +276,8 @@ function_exists "downloader" || downloader () { : "${search_result_features=}" : "${search_region:=US}" +: "${nsfw:=false}" + : "${scrape_search_exclude:= youtube-subscriptions S SI T youtube-trending H history }" @@ -696,24 +698,54 @@ scrape_peertube () { ## Odysee {{{ scrape_odysee () { page_query=$1 + [ "${#page_query}" -le 2 ] && die 4 "Odysee searches must be 3 or more characters\n" output_json_file=$2 print_info "Scraping Odysee ($page_query)\n" _tmp_json="${session_temp_dir}/odysee.json" - # TODO: filters - _get_request "https://lighthouse.lbry.com/search" -G \ - --data-urlencode "s=$page_query" \ - --data-urlencode "mediaType=video,audio" \ - --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \ - --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?" + case "$search_sort_by" in + upload_date|newest_first) search_sort_by="release_time" ;; + oldest_first) search_sort_by="^release_time" ;; + relevance) search_sort_by="" ;; + esac + case "$search_upload_date" in + week|month|year) search_upload_date="this${search_upload_date}" ;; + day) search_upload_date="today" ;; + esac + + case "$nsfw" in + 1) nsfw=true ;; + 0) nsfw=false ;; + esac + + #this if is because when search_sort_by is empty, it breaks lighthouse + if [ -n "$search_sort_by" ]; then + _get_request "https://lighthouse.lbry.com/search" -G \ + --data-urlencode "s=$page_query" \ + --data-urlencode "mediaType=video,audio" \ + --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \ + --data-urlencode "sort_by=$search_sort_by" \ + --data-urlencode "time_filter=$search_upload_date" \ + --data-urlencode "nsfw=$nsfw" \ + --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?" + else + _get_request "https://lighthouse.lbry.com/search" -G \ + --data-urlencode "s=$page_query" \ + --data-urlencode "mediaType=video,audio" \ + --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \ + --data-urlencode "time_filter=$search_upload_date" \ + --data-urlencode "nsfw=$nsfw" \ + --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?" + fi + #select(.duration != null) selects videos that aren't live, there is no .is_live key jq ' def pad_left(n; num): num | tostring | if (n > length) then ((n - length) * "0") + (.) else . end ; - [ .[] | + [ .[] |select(.duration != null) | { ID: .claimId, title: .title, @@ -725,7 +757,6 @@ scrape_odysee () { date: .cq_created_at } ]' < "$_tmp_json" >> "$output_json_file" - # TODO: error handling 2 character } ## }}} @@ -1179,7 +1210,7 @@ parse_opt () { #for some reason optarg may equal opt intentionally, #this checks the unmodified optarg, which will only be equal if there is no = sign [ "$opt" = "$OPTARG" ] && optarg="" - function_exists "on_opt_parse" && on_opt_parse "$opt" "$optarg" "$OPT" "$OPTARG" || return 0 + function_exists "on_opt_parse" && { on_opt_parse "$opt" "$optarg" "$OPT" "$OPTARG" || return 0; } case $opt in h|help) usage; exit 0 ;; D|external-menu) [ -z "$optarg" ] || [ $optarg -eq 1 ] && interface='ext_menu' ;; @@ -1230,6 +1261,7 @@ parse_opt () { url-handler) url_handler="${optarg:-multimedia_player}" ;; keep-cache) keep_cache="${optarg:-1}" ;; submenu-opts) submenu_opts="${optarg:-1}" ;; + nsfw) nsfw="${optarg:-true}" ;; *) [ "$OPT" = "-" ] && print_info "$0: illegal long option -- $opt\n";; esac @@ -1372,22 +1404,8 @@ scrape_website () { ;; *) #custom scrapers {{{ - - # Config parsing {{{ - set -- - [ -f "$YTFZF_CUSTOM_SCRAPERS_DIR/$scrape_type.conf" ] && { - custom_scraper_variables="$(sed '/^vars:/s/^vars: //' "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}.conf")" - #this sets each variable's value (even with spaces) to it's own field - IFS=' ' - for var in $(printf "%s " $custom_scraper_variables); do - set -- "$@" "$(IFS= eval printf $var)" - done - unset var - } - #}}} - - #if the file doesn't exist, it will exit with code 127 - "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" "$search" "$ytfzf_video_json_file" "$@" + [ -f "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" ] && . "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" || return 127 + scrape_$(printf "%s" "$scrape_type" | sed 's/-/_/g') "$search" "$ytfzf_video_json_file" || return "$?" #}}} esac rv="$?" @@ -1395,11 +1413,29 @@ scrape_website () { return $rv } +total_search_sort_by="$search_sort_by" +total_search_upload_date="$search_upload_date" + +manage_multi_filters () { + #if this is empty search_sort_by will be set to empty which isn't what we want + [ -n "$total_search_sort_by" ] && { + search_sort_by="${total_search_sort_by%%,*}" + total_search_sort_by="${total_search_sort_by#*,}" + } + [ -n "$total_search_upload_date" ] && { + search_upload_date="${total_serach_upload_date%%,*}" + total_search_upload_date="${total_search_upload_date#*,}" + } + #for custom scrapers + function_exists "manage_multi_custom_filters" && manage_multi_custom_filters +} + IFS="," set -f for curr_scrape in $scrape; do #only ask for search if it's empty and scrape isn't something like S or T printf "%s" "$scrape_search_exclude" | grep -Fqv " $curr_scrape " && { [ "$search" = "-" ] || [ -z "$search" ]; } && search_prompt_menu + manage_multi_filters function_exists "on_search" && on_search "$search" "$curr_scrape" scrape_website "$curr_scrape" "$search" handle_scrape_error "$?" @@ -1427,17 +1463,8 @@ while :; do "") interface_text "$ytfzf_video_json_file" "$ytfzf_selected_urls" ;; *) # custom interfaces {{{ - #incase there are no extra variables wanted - set -- - [ -f "$YTFZF_CUSTOM_SCRAPERS_DIR/$interface.conf" ] && { - custom_interface_variables="$(sed '/^vars:/s/^vars: //' "${YTFZF_CUSTOM_SCRAPERS_DIR}/${interface}.conf")" - IFS=' ' - for var in $(printf "%s " $custom_interface_variables); do - set -- "$@" "$(IFS= eval printf $var)" - done - unset var - } - "$YTFZF_CUSTOM_INTERFACE_DIR/$interface" "$ytfzf_video_json_file" "$ytfzf_selected_urls" "$@";; + [ -f "${YTFZF_CUSTOM_INTERFACES_DIR}/${interface}" ] && . "$YTFZF_CUSTOM_INTERFACES_DIR/$interface" || die 2 "interface: \"$interface\" does not exist\n" + interface_$(printf "%s" "$interface" | sed 's/-/_/g') "$ytfzf_video_json_file" "$ytfzf_selected_urls" ;; #}}} esac handle_actions < "$ytfzf_selected_urls"