Merge branch 'rewrite'

docs/man/ytfzf.1

@@ -200,7 +200,7 @@ Scrapers
 .TP
 .BI \-c " scrapers" ", " "\-\-scrape=scrapers"
 Set which scraper to use.
-Multiple scrapers can be seperated by ,
+Multiple scrapers can be separated by ,
 The currently supported builtin scrapers are:
 .RS
 .TP
@@ -270,7 +270,7 @@ convert the invidious links to youtube links before playing/downloading.
 Scraper Options
 .RS
 .PP
-Currently, \-\-sort\-by, \-\-upload\-date, \-\-video\-duration, \-\-type, \-\-thumbnail\-quality, and \-\-features only applies to the scrape: youtube/Y
+Currently, \-\-video\-duration, \-\-type, \-\-thumbnail\-quality, and \-\-features only applies to the scrape: youtube/Y
 .TP
 .BR \-D ", " \-\-external\-menu
 Use an external menu instead of fzf.
@@ -288,21 +288,38 @@ Amount of videos to scrape on odysee.
 This can also be set in the config file with
 .BR odysee_video_search_count .
 .TP
+.BR "\-\-nsfw"
+Whether or not to search for nsfw videos.
+.br
+Only works with odysee/O
+This can also be set in the config file with
+.BR nsfw .
+.TP
 .BI "\-\-sort\-by=sort"
+Works with youtube/Y and odysee/O.
+.br
+To use a different sort for each scrape, use a "," to separate the sorts.
+.br
 As apposed to \-\-sort, this happens during the serach, not after.
 Results should sort by:
 .RS
 .TP
 .IR relevance
 .TP
-.IR rating
+.IR rating " (youtube only)"
 .TP
 .IR upload_date
 .TP
-.IR view_count
+.IR oldest_first " (odysee only)"
+.TP
+.IR view_count " (youtube only)"
 .RE
 .TP
 .BI "\-\-upload\-date=time\-frame"
+Works with, youtube/Y and odysee/O
+.br
+To use a different sort for each scrape, use a "," to separate the dates.
+.br
 Search for videos within the last:
 .RS
 .TP
@@ -368,7 +385,7 @@ The end frame of the video (low quality)
 .RE
 .TP
 .BI "\-\-features=features"
-The features to have on a video (comma seperated).
+The features to have on a video (comma separated).
 .RS
 .TP
 .IR hd

docs/man/ytfzf.5

@@ -422,11 +422,13 @@ The attribute to sort by when searching.
 .TP
 .IR relevance " (default)"
 .TP
-.IR rating
+.IR rating " (youtube only)"
 .TP
 .IR upload_date
 .TP
-.IR view_count
+.IR oldest_first " (odysee only)"
+.TP
+.IR view_count " (youtbe only)"
 .RE
 
 .TP
@@ -470,6 +472,12 @@ The type of results to get.
 .IR all " (may not work on some instances)"
 .RE
 
+.TP
+.RB $ nsfw
+Whether or not to search for nsfw videos in odysee/O.
+.br
+.RI default: " false"
+
 .TP
 .RB $ search_result_features
 The features to have on a video (comma seperated).
@@ -873,6 +881,16 @@ This function takes 1 argument:
     The action.
 .EE
 
+.TP
+.BR manage_multi_custom_filters ()
+This function should add the ability for filters to be used in multiple scrapes,
+.br
+To see how this is done, take a look at the manage_multi_filters() function in ytfzf.
+.br
+This function is called before the website is scraped.
+.br
+This function takes no arguments.
+
 .RE
 
 .SH VIDEO JSON FORMAT
@@ -922,9 +940,18 @@ To easily get the formated json for a video, run
 
 .SH CUSTOM SCRAPERS
 .PP
-Custom scrapers are programs located in $YTFZF_CUSTOM_SCRAPERS_DIR.
+Custom scrapers shell scripts located in $YTFZF_CUSTOM_SCRAPERS_DIR.
+.br
+A scraper is responsible for scraping videos from a website and APPENDING them to "$ytfzf_video_json_file".
+.br
+The shell script must be the same shell as your /bin/sh.
 .br
-To customize some behavior of the scraper see CUSTOM SCRAPER CONFIG
+In addition, the script must also define the function
+.I interface_<name_of_interface>
+.br
+With _ replacing \-.
+.br
+This function could handle everything itself, or call another program written in any language to handle it.
 .RE
 .PP
 A custom scraper will take the search query as the first argument to the program
@@ -936,70 +963,27 @@ The JSON should be structured as described in VIDEO JSON FORMAT
 The final JSON shall be
 .B APPENDED
 to the file given as an argument.
-.PP
-A custom scraper can be written in any programming language so long as the file can be run as a normal command
 .RE
 
-.SH CUSTOM SCRAPER CONFIG
+.SH CUSTOM INTERFACES
 .PP
-To customize some behavior of a scraper create a file named <scraper-name>.conf. (replace <scraper-name> with the name of the scraper)
+Custom interfaces are shell scripts located in $YTFZF_CUSTOM_INTERFACES_DIR.
 .br
-This file would also be located in $YTFZF_CUSTOM_SCRAPER_DIR.
-.PP
-The syntax of this file is as such:
-.RS
-.EX
-key: value
-key2: value2
-.EE
-.RE
-There are no quotations around anything.
-
-.SS CUSTOM SCRAPER CONFIG OPTIONS
-.PP
-Options to use as keys in <scraper-name>.conf
-.TP
-.RB vars
-The variables to pass into the scraper after the search and json file path.
+An interface is responsible for letting the user pick a video from "$ytfzf_video_json_file", then writing the url(s) to "$ytfzf_selected_urls"
 .br
-Example:
-.RS
-.EX
-vars: $is_auto_select $is_sort
-.EE
-This will make the 3rd argument the value of $is_auto_select, and the 4th $is_sort
-.RE
-
-.SH CUSTOM INTERFACES
-.PP
-Custom interfaces are programs located in $YTFZF_CUSTOM_INTERFACES_DIR.
+The shell script must be the same shell as your /bin/sh.
 .br
-To customize some behavior of the interface see CUSTOM INTERFACE CONFIG
-.RE
-.PP
-A custom interface will take a path to the json file holding all data about all the videos as the first argument.
+In addition, the script must also define the function
+.I interface_<name_of_interface>
 .br
-The second argument will be a path to a file to store the selected url in, separated by new lines.
-
-.SH CUSTOM INTERFACE CONFIG
-.PP
-Similar to CUSTOM SCRAPERS, some behavior of an interface can be changed in <interface-name>.conf
+With _ replacing \-.
 .br
-This file should also be in $YTFZF_CUSTOM_INTERFACES_DIR.
+This function could handle everything itself, or call another program written in any language to handle it.
+.RE
 .PP
-The syntax is the same as CUSTOM SCRAPER CONFIG
-
-.SS CUSTOM INTERFACE CONFIG OPTIONS
-.TP
-.RB vars
-The variabels to pass into the interface after the 2 file paths.
+interface_<name_of_interface> will take a path to the json file holding all data about all the videos as the first argument.
 .br
-Example:
-.RS
-.EX
-vars: $show_thumbnails $is_sort
-.EE
-This will make the 3rd argument the value of $show_thumbnails, and the 4th, $is_sort
+The second argument will be a path to a file to store the selected url in, separated by new lines.
 
 .SH THUMBNAIL VIEWERS
 Custom thumbnail viewers are programs in $YTFZF_THUMBNAIL_VIEWERS_DIR.

ytfzf

@@ -136,7 +136,7 @@ dep_check "jq" || die 3 "jq is a required dependancy, please install it\n"
 : "${YTFZF_CONFIG_FILE:=$YTFZF_CONFIG_DIR/conf.sh}"
 : "${YTFZF_SUBSCRIPTIONS_FILE:=$YTFZF_CONFIG_DIR/subscriptions}"
 : "${YTFZF_THUMBNAIL_VIEWERS_DIR:=$YTFZF_CONFIG_DIR/thumbnail-viewers}"
-: "${YTFZF_CUSTOM_INTERFACE_DIR:=$YTFZF_CONFIG_DIR/interfaces}"
+: "${YTFZF_CUSTOM_INTERFACES_DIR:=$YTFZF_CONFIG_DIR/interfaces}"
 
 [ -f "$YTFZF_CONFIG_FILE" ] && . "$YTFZF_CONFIG_FILE"
 #}}}
@@ -276,6 +276,8 @@ function_exists "downloader" || downloader () {
 : "${search_result_features=}"
 : "${search_region:=US}"
 
+: "${nsfw:=false}"
+
 : "${scrape_search_exclude:= youtube-subscriptions S SI T youtube-trending H history }"
 
 
@@ -696,24 +698,54 @@ scrape_peertube () {
 ## Odysee {{{
 scrape_odysee () {
 	page_query=$1
+	[ "${#page_query}" -le 2 ] && die 4 "Odysee searches must be 3 or more characters\n"
 	output_json_file=$2
 	print_info "Scraping Odysee ($page_query)\n"
 
 	_tmp_json="${session_temp_dir}/odysee.json"
 
-	# TODO: filters
-	_get_request "https://lighthouse.lbry.com/search" -G \
-	    --data-urlencode "s=$page_query" \
-	    --data-urlencode "mediaType=video,audio" \
-	    --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \
-	    --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?"
+	case "$search_sort_by" in
+		upload_date|newest_first) search_sort_by="release_time" ;;
+		oldest_first) search_sort_by="^release_time" ;;
+		relevance) search_sort_by="" ;;
+	esac
+	case "$search_upload_date" in
+		week|month|year) search_upload_date="this${search_upload_date}" ;;
+		day) search_upload_date="today" ;;
+	esac
+
+	case "$nsfw" in
+		1) nsfw=true ;;
+		0) nsfw=false ;;
+	esac
+
+	#this if is because when search_sort_by is empty, it breaks lighthouse
+	if [ -n "$search_sort_by" ]; then
+		_get_request "https://lighthouse.lbry.com/search" -G \
+		    --data-urlencode "s=$page_query" \
+		    --data-urlencode "mediaType=video,audio" \
+		    --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \
+		    --data-urlencode "sort_by=$search_sort_by" \
+		    --data-urlencode "time_filter=$search_upload_date" \
+		    --data-urlencode "nsfw=$nsfw" \
+		    --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?"
+	else
+		_get_request "https://lighthouse.lbry.com/search" -G \
+		    --data-urlencode "s=$page_query" \
+		    --data-urlencode "mediaType=video,audio" \
+		    --data-urlencode "include=channel,title,thumbnail_url,duration,cq_created_at,description,view_cnt" \
+		    --data-urlencode "time_filter=$search_upload_date" \
+		    --data-urlencode "nsfw=$nsfw" \
+		    --data-urlencode "size=$odysee_video_search_count" > "$_tmp_json" || return "$?"
 
+	fi
+	#select(.duration != null) selects videos that aren't live, there is no .is_live key
 	jq '
 	def pad_left(n; num):
 		num | tostring |
 			if (n > length) then ((n - length) * "0") + (.) else . end
 		;
-	[ .[] |
+	[ .[] |select(.duration != null) |
 	    {
 			ID: .claimId,
 			title: .title,
@@ -725,7 +757,6 @@ scrape_odysee () {
 			date: .cq_created_at
 	    }
 	]' < "$_tmp_json" >> "$output_json_file" 
-	# TODO:  error handling 2 character
 
 }
 ## }}}
@@ -1179,7 +1210,7 @@ parse_opt () {
 	#for some reason optarg may equal opt intentionally,
 	#this checks the unmodified optarg, which will only be equal if there is no = sign
 	[ "$opt" = "$OPTARG" ] && optarg=""
-	function_exists "on_opt_parse" && on_opt_parse "$opt" "$optarg" "$OPT" "$OPTARG" || return 0
+	function_exists "on_opt_parse" && { on_opt_parse "$opt" "$optarg" "$OPT" "$OPTARG" || return 0; }
 	case $opt in
 		h|help) usage; exit 0 ;;
 		D|external-menu) [ -z "$optarg" ] || [ $optarg -eq 1 ] && interface='ext_menu' ;;
@@ -1230,6 +1261,7 @@ parse_opt () {
 		url-handler) url_handler="${optarg:-multimedia_player}" ;;
 		keep-cache) keep_cache="${optarg:-1}" ;;
 		submenu-opts) submenu_opts="${optarg:-1}" ;;
+		nsfw)  nsfw="${optarg:-true}" ;;
 		*) 
 			[ "$OPT" = "-" ] && print_info "$0: illegal long option -- $opt\n";;
 	esac
@@ -1372,34 +1404,38 @@ scrape_website () {
 			;;
                 *)
                         #custom scrapers {{{
-
-			# Config parsing {{{
-			set --
-			[ -f "$YTFZF_CUSTOM_SCRAPERS_DIR/$scrape_type.conf" ] && {
-				custom_scraper_variables="$(sed '/^vars:/s/^vars: //' "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}.conf")"
-				#this sets each variable's value (even with spaces) to it's own field
-				IFS=' '
-				for var in $(printf "%s " $custom_scraper_variables); do
-					set -- "$@" "$(IFS= eval printf $var)"
-				done
-				unset var
-			}
-			#}}}
-
-			#if the file doesn't exist, it will exit with code 127
-			"${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" "$search" "$ytfzf_video_json_file" "$@"
+			[ -f "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" ] && . "${YTFZF_CUSTOM_SCRAPERS_DIR}/${scrape_type}" || return 127
+			scrape_$(printf "%s" "$scrape_type" | sed 's/-/_/g') "$search" "$ytfzf_video_json_file" || return "$?"
                         #}}}
 	esac
 	rv="$?"
 	unset scrape_type
 	return $rv
 }
 
+total_search_sort_by="$search_sort_by"
+total_search_upload_date="$search_upload_date"
+
+manage_multi_filters () {
+	#if this is empty search_sort_by will be set to empty which isn't what we want
+	[ -n "$total_search_sort_by" ] && {
+		search_sort_by="${total_search_sort_by%%,*}"
+		total_search_sort_by="${total_search_sort_by#*,}"
+	}
+	[ -n "$total_search_upload_date" ] && {
+		search_upload_date="${total_serach_upload_date%%,*}"
+		total_search_upload_date="${total_search_upload_date#*,}"
+	}
+	#for custom scrapers
+	function_exists "manage_multi_custom_filters" && manage_multi_custom_filters
+}
+
 IFS=","
 set -f
 for curr_scrape in $scrape; do
 	#only ask for search if it's empty and scrape isn't something like S or T
 	printf "%s" "$scrape_search_exclude" | grep -Fqv " $curr_scrape " && { [ "$search" = "-" ] || [ -z "$search" ]; } && search_prompt_menu
+	manage_multi_filters
 	function_exists "on_search" && on_search "$search" "$curr_scrape"
 	scrape_website "$curr_scrape" "$search"
 	handle_scrape_error "$?"
@@ -1427,17 +1463,8 @@ while :; do
 		 "") interface_text       "$ytfzf_video_json_file" "$ytfzf_selected_urls" ;;
 		 *) 
 			 # custom interfaces {{{
-			 #incase there are no extra variables wanted
-			 set --
-			 [ -f "$YTFZF_CUSTOM_SCRAPERS_DIR/$interface.conf" ] && {
-				custom_interface_variables="$(sed '/^vars:/s/^vars: //' "${YTFZF_CUSTOM_SCRAPERS_DIR}/${interface}.conf")"
-			 	IFS=' '
-				for var in $(printf "%s " $custom_interface_variables); do
-					set -- "$@" "$(IFS= eval printf $var)"
-				done
-				unset var
-			 }
-			 "$YTFZF_CUSTOM_INTERFACE_DIR/$interface" "$ytfzf_video_json_file" "$ytfzf_selected_urls" "$@";;
+			 [ -f "${YTFZF_CUSTOM_INTERFACES_DIR}/${interface}" ] && . "$YTFZF_CUSTOM_INTERFACES_DIR/$interface" || die 2 "interface: \"$interface\" does not exist\n"
+			 interface_$(printf "%s" "$interface" | sed 's/-/_/g') "$ytfzf_video_json_file" "$ytfzf_selected_urls" ;;
 			 #}}}
 	esac
 	handle_actions < "$ytfzf_selected_urls"