# shellcheck shell=dash disable=SC2154,SC2034

# using the search to query all data

# x curl -svvv -w "data: ${#1}\n" -G "https://api.shodan.io/tools/httpheaders?data=[1-3]" --data-urlencode a=3  -o /dev/null 2>a.out
___x_cmd_shodan_host_download(){
    local X_help_cmd='x help -m shodan host download';   help:arg:parse
    local limit=1000
    local IFS=" "
    local query=""
    local file=""

    while [ $# -gt 0 ]; do
        case "$1" in
            -a|--all)       limit=-1;       shift 1 ;;
            --limit)        limit="$2";     shift 2 ;;
            -f|--file)      file="$2";      shift 2 ;;
            *)              query="$*";     break   ;;
        esac
    done

    [ -n "$file" ] || file="x_shodan_data.json"
    ! [ -f "$file" ] || N=shodan M="File already exists: $file" log:ret:1


    local x_=; ___x_cmd_shodan_host_download___count "$@"; local page="$x_"
    local tmpdirp=; tmpdirp="$___X_CMD_ROOT_TMP/shodan/.x-cmd.shodan.dir"

    ___x_cmd_shodan_host_download___raw "$page" | ___x_cmd_shodan_host_download___handle_data "$page" |\
        x ourl pbar --total "${page}" --size "4"
}

___x_cmd_shodan_host_download___handle_data(){
    local page="$1"
    ourl:writeout:init

    while ourl:writeout:read; do
        printf "%s\n" "[http_code=${XOW_code}]" "[size=${XOW_size} bytes]" "[speed=${XOW_speed} bytes]" "[exitcode=${XOW_exitcode}]"
        page="$((page-1))"
        case "${XOW_code}" in
            20*)        x jq '.matches[]' >> "$file" <"${XOW_fp}" ;;
            *)          N=shodan M="[Http_code=$XOW_code] Get data failed" log:ret:1 ;;
        esac
    done
    x rmrf "${tmpdirp}"
    [ "${page}" -le 0 ]
}

___x_cmd_shodan_host_download___raw(){
    local page="$1"; shift

    x mkdirp "$tmpdirp"

    set -- "/shodan/host/search?page=[1-$page]" --output "$tmpdirp/#1" --fail-early --fail-with-body
    set -- "$@" --no-buffer -w "$___X_CMD_OURL_WRITEOUT"
    set -- "$@" --data-urlencode "query=$query"

    local ___X_CMD_OURL_STDERR2STDOUT=1
    ___x_cmd_shodan_ourl get  "$@"
}

___x_cmd_shodan_host_download___count(){
    local count=
    count=$( ___x_cmd_shodan_host_count "$@" | x jo . .total) || shodan:data:ret
    x is int "$count" || N=shodan M="Fail to fetch count=$count" log:ret:1
    shodan:info "total [count=$count]"

    if [ "$limit" -gt 0 ] && [ "$limit" -lt "$count" ]; then
        shodan:info "Download only top $limit with Limit=$limit"
        count="$limit"
    fi

    x_="$((  (count-1)/100 + 1  ))"
    shodan:info "Number of pages is $x_"
    shodan:info "Target file ==> $file"
}


