preprocessing_zepplin.json

{"paragraphs":[{"text":"%spark.pyspark\nfrom pyspark.sql import SQLContext\nimport os\nimport re\nimport sys\nimport unicodedata\nimport itertools\nfrom pyspark.sql import SQLContext\n\nsc._jsc.hadoopConfiguration().set(\"fs.s3a.access.key\", \"AKIAISNUXXLZRX55BRUA\")\nsc._jsc.hadoopConfiguration().set(\"fs.s3a.secret.key\", \"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\")\nsc._jsc.hadoopConfiguration().set(\"fs.s3a.endpoint\", \"s3.amazonaws.com\")\nsqlContext = SQLContext(sc)\nimport random\ndfParquet = sqlContext.read.parquet(\"s3a://msdbucket/preprocess_final\")\n# dfMSD.printSchema()","user":"anonymous","dateUpdated":"2020-05-02T20:52:52+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456315_755440074","id":"20200418-094817_231359692","dateCreated":"2020-05-02T18:50:56+0000","dateStarted":"2020-05-02T20:52:52+0000","dateFinished":"2020-05-02T20:53:21+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:2846"},{"text":"%spark.pyspark\n# ROTATION SYMBOLS (A and B => B and A)\nrotation_symbols = ['\\|', '/', '&', ',', '\\+', ';', '_']#, '\\-']\nrotation_words = ['and', 'y', 'et', 'vs', 'vs.', 'v', 'with', 'feat',\n                  'feat.', 'featuring', 'presents', 'ft.', 'pres.']\n\n# SYMBOLS TO REMOVE AT THE BEGINNING\nstub_to_remove = ['dj', 'dj.', 'mc', 'm.c.', 'mc.', 'the', 'los', 'les']\n\n# SYMBOLS TO REMOVE AT THE END\nend_to_remove1 = ['big band', 'trio', 'quartet', 'ensemble', 'orchestra']\nend_to_remove2 = ['band']\n\n# COMPILED REGULAR EXPRESSION\n# white spaces\nre_space = re.compile(r'\\s')\n# non alphanumeric\nre_nonalphanum = re.compile(r'\\W')\n# rotation symbols\nre_rotsymbols = re.compile('\\s*?' + '|'.join(rotation_symbols) + '\\s*?')\n# rotation words\nre_rotwords = re.compile(r'\\s(' + '|'.join(rotation_words) + ')\\s')\n# stub to remove\nre_remstub = re.compile('(' + '|'.join(stub_to_remove) + ')\\s(.*)')\n# ending to remove\nre_remending1 = re.compile('(.*)\\s(' + '|'.join(end_to_remove1) + ')')\nre_remending2 = re.compile('(.*)\\s(' + '|'.join(end_to_remove2) + ')')\n# quotes to remove\nre_remquotes = re.compile('(.+)\\s(\".+?\")\\s(.+)')\n# parenthesis to remove\nre_remparenthesis = re.compile('(.+)\\s(\\(.+?\\))\\s*(.*)')\n# brackets to remove\nre_rembrackets = re.compile('(.+)\\s(\\[.+?\\])\\s*(.*)')\n\n\ndef char_is_ascii(c):\n    \"\"\"\n    Check if a unicode character, e.g. u'A', u'1' or u'\\u0301' is ASCII\n    \"\"\"\n    #return ord(c) < 128\n    # the following should be faster, according to:\n    #http://stackoverflow.com/questions/196345/how-to-check-if-a-string-in-python-is-in-ascii\n    return c < u\"\\x7F\"\n\n\ndef remove_non_ascii(s):\n    \"\"\"\n    Normalize characters in unicode string 's' that are not ASCII,\n    try to transform accented characters to non accented version.\n    Otherwise, remove non-ascii chars\n    \"\"\"\n    decomposition = unicodedata.normalize('NFKD', s)\n    return \"\".join(list(filter(lambda x: char_is_ascii(x), decomposition)))\n\n\ndef to_lower_case(s):\n    \"\"\"\n    transform a unicode string 's' to lowercase\n    ok, this one is trivial, I know\n    \"\"\"\n    return s.lower()\n\n\ndef remove_spaces(s):\n    \"\"\"\n    Remove all possible spaces in the unicode string s\n    \"\"\"\n    return re_space.sub('', s)\n\n\ndef replace_rotation_symbols(s):\n    \"\"\"\n    Mostly, replace '&' by 'and'\n    \"\"\"\n    return re_rotsymbols.sub(' and ', s)\n\n\ndef remove_stub(s):\n    \"\"\"\n    Remove a questionable beginning, e.g. dj\n    otherwise return string at is\n    \"\"\"\n    m = re_remstub.match(s)\n    if not m:\n        return s\n    return m.groups()[1]\n\n\ndef remove_endings(s):\n    \"\"\"\n    Remove questionable endings, e.g. 'band'\n    \"\"\"\n    m = re_remending1.match(s)\n    if m:\n       s = m.groups()[0]\n    m = re_remending2.match(s)\n    if m:\n        s = m.groups()[0]\n    return s\n\n\ndef remove_quotes(s):\n    \"\"\"\n    Remove the quote, like Thierry \"The Awesomest\" BM\n    \"\"\"\n    m = re_remquotes.match(s)\n    if not m:\n        return s\n    parts = m.groups()\n    assert len(parts) == 3\n    return parts[0] + ' ' + parts[2]\n\n\ndef remove_parenthesis(s):\n    \"\"\"\n    Remove parenthesis, like Thierry (Coolest guy)\n    \"\"\"\n    print(s)\n    m = re_remparenthesis.match(s)\n    if not m:\n        return s\n    parts = m.groups()\n    assert len(parts) >= 2\n    if len(parts) == 2:\n        return parts[0]\n    return parts[0] + ' ' + parts[2]\n\n\ndef remove_brackets(s):\n    \"\"\"\n    Remove brackets, like Thierry [Coolest guy]\n    \"\"\"\n    m = re_rembrackets.match(s)\n    if not m:\n        return s\n    parts = m.groups()\n    assert len(parts) >= 2\n    if len(parts) == 2:\n        return parts[0]\n    return parts[0] + ' ' + parts[2]\n\n\ndef normalize_no_rotation(s):\n    \"\"\"\n    We normalize a name that is supposed to contain no\n    rotation term ('and', 'y', ...)\n    \"\"\"\n    # remove beginning\n    s = remove_stub(s)\n    # remove ends\n    s = remove_endings(s)    \n    # remove ()\n    s = remove_parenthesis(s)\n    # remove \"\"\n    s = remove_quotes(s)\n    return s\n\n\ndef split_rotation_words(s):\n    \"\"\"\n    Split a name using the rotation words: 'and', 'vs', 'y', 'et', ...\n    then create all possible permutations\n    \"\"\"\n    parts = re_rotwords.split(s)\n    parts = list(filter(lambda p: not p in rotation_words, parts))[:5]\n    results = set()\n    # keep only the individual elems (risky?)\n    for p in parts:\n        results.add(p)\n    # create all permutations\n    permutations = itertools.permutations(parts)\n    #maxperm = 30\n    #count_perm = 0\n    for perm in permutations:\n        #count_perm += 1\n        #if count_perm > maxperm:\n        #    break\n        results.add(' '.join(perm))\n    # redo the same but remove the stub first for all parts\n    parts = list(map(lambda p: normalize_no_rotation(p), parts))\n    for p in parts:\n        results.add(p)\n    permutations = itertools.permutations(parts)\n    for perm in permutations:\n        results.add(' '.join(perm))\n    # done\n    return results\n\n\ndef remove_nonalphanumeric(s):\n    \"\"\"\n    Remove usual punctuation signs:  ! , ? : ; . '   etc\n    Also, we transform long spaces into normal ones\n    \"\"\"\n    # split around non-alphanum chars\n    parts = re_nonalphanum.split(s)\n    # remove empty spots\n    parts = list(filter(lambda p: p, parts))\n    # rejoin with regular space ' '\n    return ' '.join(parts)\n\n\ndef normalize_artist(s):\n    \"\"\"\n    Return a set of normalized versions of that artist name\n    \"\"\"\n    # normalized versions\n    results = set()\n    # lower case\n    s = to_lower_case(s)\n    results.add(s)\n    print(s)\n    # remove non-ascii chars (try to replace them)\n    s = remove_non_ascii(s)\n    print(s)\n    results.add(s)\n    # try removing parenthesis before, in case there's an & in it\n    s2 = remove_parenthesis(s)\n    results.add(s2)\n    # replace rotation symbols\n    s = replace_rotation_symbols(s)\n    # split and permute according to rotation words\n    permutations = split_rotation_words(s)\n    results.update(permutations)\n    print(s)\n    # remove non-alphanumeric and normalize spaces\n    results = list(map(lambda s: remove_nonalphanumeric(s), results))\n    # remove all spaces\n    results = list(map(lambda s: remove_spaces(s), results))\n    # done (and remove dupes)\n    return set(results)\n\n\ndef normalize_title(s):\n    \"\"\"\n    Return a set of normalized versions of that title\n    \"\"\"\n    # normalized versions\n    results = set()\n    # lower case\n    s = to_lower_case(s)\n    results.add(s)\n    # remove non-ascii chars (try to replace them)\n    s = remove_non_ascii(s)\n    results.add(s)\n    # try removing parenthesis\n    s = remove_parenthesis(s)\n    results.add(s)\n    # try removing brackets\n    s = remove_brackets(s)\n    results.add(s)\n    # remove non-alphanumeric and normalize spaces\n    results = map(lambda s: remove_nonalphanumeric(s), results)\n    # remove all spaces\n    results = map(lambda s: remove_spaces(s), results)\n    # done (and remove dupes)\n    return set(results)","user":"anonymous","dateUpdated":"2020-05-02T20:55:07+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456319_269493966","id":"20200429-234832_1238269878","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2847","dateFinished":"2020-05-02T20:55:07+0000","dateStarted":"2020-05-02T20:55:07+0000"},{"text":"%spark.pyspark\nimport pyspark.sql.functions as F\nimport pyspark.sql.types as T\n\ndef normalize_name_udf(col):\n    return sorted(list(normalize_artist(col)), key=len)[0]\n\ndef normalize_title_udf(col):\n    return sorted(list(normalize_title(col)), key=len)[0]\n    \ndef normalize(col):\n    return col.lower()\n\ndef group_normalize(col):\n    return [c.lower() for c in col]\n\n# if we assume that my_func returns a string\nudf_name = F.UserDefinedFunction(normalize_name_udf, T.StringType())\nudf_title = F.UserDefinedFunction(normalize_title_udf, T.StringType())\nudf_lower = F.UserDefinedFunction(normalize, T.StringType())\nudf_group_lower = F.UserDefinedFunction(group_normalize)\n\ndfParquet = dfParquet.withColumn('artist_name', udf_name('artist_name'))\ndfParquet = dfParquet.withColumn('title', udf_title('title'))\ndfParquet = dfParquet.withColumn('artist_mbtags', udf_group_lower('artist_mbtags'))\ndfParquet = dfParquet.withColumn('artist_terms', udf_group_lower('artist_terms'))\ndfParquet = dfParquet.withColumn('release', udf_title('release'))\ndfParquet = dfParquet.withColumn('artist_id', udf_lower('artist_id'))\ndfParquet = dfParquet.withColumn('song_id', udf_lower('song_id'))\ndfParquet = dfParquet.withColumn('track_id', udf_lower('track_id'))\ndfParquet = dfParquet.withColumn('artist_location', udf_lower('artist_location'))\ndfParquet = dfParquet.withColumn('similar_artists', udf_group_lower('similar_artists'))","user":"anonymous","dateUpdated":"2020-05-02T20:55:27+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456320_1727744187","id":"20200429-234903_1502175902","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2848","dateFinished":"2020-05-02T20:55:27+0000","dateStarted":"2020-05-02T20:55:27+0000"},{"text":"%spark.pyspark\nfrom pyspark.ml.feature import FeatureHasher\nfrom pyspark.sql.types import *\n\n# process string features and save them in the column string_features which is of type SparseVector\n\n#dataset = dfParquet.select(['artist_name', 'title', 'release', 'artist_id', 'song_id', 'track_id', 'artist_location']) \nhasher = FeatureHasher(inputCols=['artist_name', 'title', 'release', 'artist_id', 'song_id', 'track_id', 'artist_location'],\n                       outputCol=\"string_features\")\ndf_str = hasher.transform(dfParquet)","user":"anonymous","dateUpdated":"2020-05-02T18:53:33+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456320_873811342","id":"20200501-164201_1168693960","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2849","dateFinished":"2020-05-02T18:53:34+0000","dateStarted":"2020-05-02T18:53:33+0000"},{"text":"%spark.pyspark\n# flatten array features\n\n#segments_confidence: array<float>\n#segments_loudness_max: array<float>\n#segments_loudness_max_time: array<float>\n#segments_pitches: array<array<float>>\n#segments_timbre: array<array<float>>\nfrom pyspark.sql.functions import udf\nfrom functools import reduce\nfrom pyspark.sql import functions as f\nfrom pyspark.ml.linalg import Vectors, VectorUDT\nfrom pyspark.sql.types import *\nfrom pyspark.sql.functions import explode\n\n# flatten array features\ndef vectorizer_1d(x):\n    return Vectors.dense(x)\n    \n#def vectorizer_2d(x):\n#    x_arr = np.array(x)\n#    s = np.shape(x_arr)\n#    return Vectors.dense(np.reshape(x_arr, (s[0]*s[1],)))\n\ndef vectorizer_2d(x):\n    flatten = lambda l: [item for sublist in l for item in sublist]\n    return Vectors.dense(reduce(lambda m,n: m+n, x))\n\nudf_vectorizer_1d = udf(vectorizer_1d, VectorUDT())\nudf_vectorizer_2d = udf(vectorizer_2d, VectorUDT())\n\n#df_flat = df_str.withColumn(\"segments_pitches\", udf_vectorizer_2d(\"segments_pitches\"))\n#df_flat = df_flat.withColumn(\"segments_timbre\", udf_vectorizer_2d(\"segments_timbre\"))\n#df_flat = df_flat.withColumn(\"segments_loudness_max_time\", udf_vectorizer_1d(\"segments_loudness_max_time\"))\n#df_flat = df_flat.withColumn(\"segments_loudness_max\", udf_vectorizer_1d(\"segments_loudness_max\"))\n#df_flat = df_flat.withColumn(\"segments_confidence\", udf_vectorizer_1d(\"segments_confidence\"))\n\n\n# vectors to array\n#udf_vec2arr = udf(lambda row: row.toArray().tolist(), ArrayType(DoubleType()))\n#df_flat = df_flat.withColumn(\"segments_pitches\", udf_vec2arr(\"segments_pitches\"))\n#df_flat = df_flat.withColumn(\"segments_timbre\", udf_vec2arr(\"segments_timbre\"))\n\ndf_flat = dfParquet.select(\"*\")\n\ncols = []\nfor i in range(12):\n    for j in range(10):\n        cols.append(df_flat.segments_pitches[i][j])\n\nfor i in range(12):\n    for j in range(10):\n        cols.append(df_flat.segments_timbre[i][j])\n\n\n#cols = [df_flat.segments_pitches[i] for i in range(120)]\n#cols.extend([df_flat.segments_timbre[i] for i in range(120)])\ncols.extend([df_flat.segments_loudness_max_time[i] for i in range(10)])\ncols.extend([df_flat.segments_loudness_max[i] for i in range(10)])\ncols.extend([df_flat.segments_confidence[i] for i in range(10)])\ncols.extend(['artist_familiarity', 'danceability', 'duration', 'end_of_fade_in', 'key', 'key_confidence', 'loudness', 'mode', 'song_id', 'tempo', 'time_signature', 'time_signature_confidence', 'track_id'])\n\n\ndf_flat = df_flat.select(cols)\n\nprint(df_flat)\n\n#df_flat = df_flat.withColumn(\"segments_loudness_max_time\", udf_vec2arr(\"segments_loudness_max_time\"))\n#df_flat = df_flat.withColumn(\"segments_loudness_max\", udf_vec2arr(\"segments_loudness_max\"))\n#df_flat = df_flat.withColumn(\"segments_confidence\", udf_vec2arr(\"segments_confidence\"))\n\n#for i in range(10):\n#    udf_extract = udf(lambda row: row[i], DoubleType())\n#    df_flat = df_flat.withColumn(\"segments_loudness_max_time_\" + str(i), udf_extract(\"segments_loudness_max_time\"))\n#    df_flat = df_flat.withColumn(\"segments_loudness_max_\" + str(i), udf_extract(\"segments_loudness_max\"))\n#    df_flat = df_flat.withColumn(\"segments_confidence_\" + str(i), udf_extract(\"segments_confidence\"))\n\n#for i in range(120):\n#    udf_extract = udf(lambda row: row[i], DoubleType())\n#    df_flat = df_flat.withColumn(\"segments_pitches_\" + str(i), udf_extract(\"segments_pitches\"))\n#    df_flat = df_flat.withColumn(\"segments_timbre_\" + str(i), udf_extract(\"segments_timbre\"))\n\n\n\n\n","user":"anonymous","dateUpdated":"2020-05-02T20:55:36+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"DataFrame[segments_pitches[0][0]: float, segments_pitches[0][1]: float, segments_pitches[0][2]: float, segments_pitches[0][3]: float, segments_pitches[0][4]: float, segments_pitches[0][5]: float, segments_pitches[0][6]: float, segments_pitches[0][7]: float, segments_pitches[0][8]: float, segments_pitches[0][9]: float, segments_pitches[1][0]: float, segments_pitches[1][1]: float, segments_pitches[1][2]: float, segments_pitches[1][3]: float, segments_pitches[1][4]: float, segments_pitches[1][5]: float, segments_pitches[1][6]: float, segments_pitches[1][7]: float, segments_pitches[1][8]: float, segments_pitches[1][9]: float, segments_pitches[2][0]: float, segments_pitches[2][1]: float, segments_pitches[2][2]: float, segments_pitches[2][3]: float, segments_pitches[2][4]: float, segments_pitches[2][5]: float, segments_pitches[2][6]: float, segments_pitches[2][7]: float, segments_pitches[2][8]: float, segments_pitches[2][9]: float, segments_pitches[3][0]: float, segments_pitches[3][1]: float, segments_pitches[3][2]: float, segments_pitches[3][3]: float, segments_pitches[3][4]: float, segments_pitches[3][5]: float, segments_pitches[3][6]: float, segments_pitches[3][7]: float, segments_pitches[3][8]: float, segments_pitches[3][9]: float, segments_pitches[4][0]: float, segments_pitches[4][1]: float, segments_pitches[4][2]: float, segments_pitches[4][3]: float, segments_pitches[4][4]: float, segments_pitches[4][5]: float, segments_pitches[4][6]: float, segments_pitches[4][7]: float, segments_pitches[4][8]: float, segments_pitches[4][9]: float, segments_pitches[5][0]: float, segments_pitches[5][1]: float, segments_pitches[5][2]: float, segments_pitches[5][3]: float, segments_pitches[5][4]: float, segments_pitches[5][5]: float, segments_pitches[5][6]: float, segments_pitches[5][7]: float, segments_pitches[5][8]: float, segments_pitches[5][9]: float, segments_pitches[6][0]: float, segments_pitches[6][1]: float, segments_pitches[6][2]: float, segments_pitches[6][3]: float, segments_pitches[6][4]: float, segments_pitches[6][5]: float, segments_pitches[6][6]: float, segments_pitches[6][7]: float, segments_pitches[6][8]: float, segments_pitches[6][9]: float, segments_pitches[7][0]: float, segments_pitches[7][1]: float, segments_pitches[7][2]: float, segments_pitches[7][3]: float, segments_pitches[7][4]: float, segments_pitches[7][5]: float, segments_pitches[7][6]: float, segments_pitches[7][7]: float, segments_pitches[7][8]: float, segments_pitches[7][9]: float, segments_pitches[8][0]: float, segments_pitches[8][1]: float, segments_pitches[8][2]: float, segments_pitches[8][3]: float, segments_pitches[8][4]: float, segments_pitches[8][5]: float, segments_pitches[8][6]: float, segments_pitches[8][7]: float, segments_pitches[8][8]: float, segments_pitches[8][9]: float, segments_pitches[9][0]: float, segments_pitches[9][1]: float, segments_pitches[9][2]: float, segments_pitches[9][3]: float, segments_pitches[9][4]: float, segments_pitches[9][5]: float, segments_pitches[9][6]: float, segments_pitches[9][7]: float, segments_pitches[9][8]: float, segments_pitches[9][9]: float, segments_pitches[10][0]: float, segments_pitches[10][1]: float, segments_pitches[10][2]: float, segments_pitches[10][3]: float, segments_pitches[10][4]: float, segments_pitches[10][5]: float, segments_pitches[10][6]: float, segments_pitches[10][7]: float, segments_pitches[10][8]: float, segments_pitches[10][9]: float, segments_pitches[11][0]: float, segments_pitches[11][1]: float, segments_pitches[11][2]: float, segments_pitches[11][3]: float, segments_pitches[11][4]: float, segments_pitches[11][5]: float, segments_pitches[11][6]: float, segments_pitches[11][7]: float, segments_pitches[11][8]: float, segments_pitches[11][9]: float, segments_timbre[0][0]: float, segments_timbre[0][1]: float, segments_timbre[0][2]: float, segments_timbre[0][3]: float, segments_timbre[0][4]: float, segments_timbre[0][5]: float, segments_timbre[0][6]: float, segments_timbre[0][7]: float, segments_timbre[0][8]: float, segments_timbre[0][9]: float, segments_timbre[1][0]: float, segments_timbre[1][1]: float, segments_timbre[1][2]: float, segments_timbre[1][3]: float, segments_timbre[1][4]: float, segments_timbre[1][5]: float, segments_timbre[1][6]: float, segments_timbre[1][7]: float, segments_timbre[1][8]: float, segments_timbre[1][9]: float, segments_timbre[2][0]: float, segments_timbre[2][1]: float, segments_timbre[2][2]: float, segments_timbre[2][3]: float, segments_timbre[2][4]: float, segments_timbre[2][5]: float, segments_timbre[2][6]: float, segments_timbre[2][7]: float, segments_timbre[2][8]: float, segments_timbre[2][9]: float, segments_timbre[3][0]: float, segments_timbre[3][1]: float, segments_timbre[3][2]: float, segments_timbre[3][3]: float, segments_timbre[3][4]: float, segments_timbre[3][5]: float, segments_timbre[3][6]: float, segments_timbre[3][7]: float, segments_timbre[3][8]: float, segments_timbre[3][9]: float, segments_timbre[4][0]: float, segments_timbre[4][1]: float, segments_timbre[4][2]: float, segments_timbre[4][3]: float, segments_timbre[4][4]: float, segments_timbre[4][5]: float, segments_timbre[4][6]: float, segments_timbre[4][7]: float, segments_timbre[4][8]: float, segments_timbre[4][9]: float, segments_timbre[5][0]: float, segments_timbre[5][1]: float, segments_timbre[5][2]: float, segments_timbre[5][3]: float, segments_timbre[5][4]: float, segments_timbre[5][5]: float, segments_timbre[5][6]: float, segments_timbre[5][7]: float, segments_timbre[5][8]: float, segments_timbre[5][9]: float, segments_timbre[6][0]: float, segments_timbre[6][1]: float, segments_timbre[6][2]: float, segments_timbre[6][3]: float, segments_timbre[6][4]: float, segments_timbre[6][5]: float, segments_timbre[6][6]: float, segments_timbre[6][7]: float, segments_timbre[6][8]: float, segments_timbre[6][9]: float, segments_timbre[7][0]: float, segments_timbre[7][1]: float, segments_timbre[7][2]: float, segments_timbre[7][3]: float, segments_timbre[7][4]: float, segments_timbre[7][5]: float, segments_timbre[7][6]: float, segments_timbre[7][7]: float, segments_timbre[7][8]: float, segments_timbre[7][9]: float, segments_timbre[8][0]: float, segments_timbre[8][1]: float, segments_timbre[8][2]: float, segments_timbre[8][3]: float, segments_timbre[8][4]: float, segments_timbre[8][5]: float, segments_timbre[8][6]: float, segments_timbre[8][7]: float, segments_timbre[8][8]: float, segments_timbre[8][9]: float, segments_timbre[9][0]: float, segments_timbre[9][1]: float, segments_timbre[9][2]: float, segments_timbre[9][3]: float, segments_timbre[9][4]: float, segments_timbre[9][5]: float, segments_timbre[9][6]: float, segments_timbre[9][7]: float, segments_timbre[9][8]: float, segments_timbre[9][9]: float, segments_timbre[10][0]: float, segments_timbre[10][1]: float, segments_timbre[10][2]: float, segments_timbre[10][3]: float, segments_timbre[10][4]: float, segments_timbre[10][5]: float, segments_timbre[10][6]: float, segments_timbre[10][7]: float, segments_timbre[10][8]: float, segments_timbre[10][9]: float, segments_timbre[11][0]: float, segments_timbre[11][1]: float, segments_timbre[11][2]: float, segments_timbre[11][3]: float, segments_timbre[11][4]: float, segments_timbre[11][5]: float, segments_timbre[11][6]: float, segments_timbre[11][7]: float, segments_timbre[11][8]: float, segments_timbre[11][9]: float, segments_loudness_max_time[0]: float, segments_loudness_max_time[1]: float, segments_loudness_max_time[2]: float, segments_loudness_max_time[3]: float, segments_loudness_max_time[4]: float, segments_loudness_max_time[5]: float, segments_loudness_max_time[6]: float, segments_loudness_max_time[7]: float, segments_loudness_max_time[8]: float, segments_loudness_max_time[9]: float, segments_loudness_max[0]: float, segments_loudness_max[1]: float, segments_loudness_max[2]: float, segments_loudness_max[3]: float, segments_loudness_max[4]: float, segments_loudness_max[5]: float, segments_loudness_max[6]: float, segments_loudness_max[7]: float, segments_loudness_max[8]: float, segments_loudness_max[9]: float, segments_confidence[0]: float, segments_confidence[1]: float, segments_confidence[2]: float, segments_confidence[3]: float, segments_confidence[4]: float, segments_confidence[5]: float, segments_confidence[6]: float, segments_confidence[7]: float, segments_confidence[8]: float, segments_confidence[9]: float, artist_familiarity: double, danceability: double, duration: double, end_of_fade_in: double, key: double, key_confidence: double, loudness: double, mode: double, song_id: string, tempo: double, time_signature: bigint, time_signature_confidence: double, track_id: string]\n"}]},"apps":[],"jobName":"paragraph_1588445456320_-478679862","id":"20200501-005834_427620358","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2850","dateFinished":"2020-05-02T20:55:38+0000","dateStarted":"2020-05-02T20:55:36+0000"},{"text":"%spark.pyspark\ndef rename_col(col_name):\n    if '[' in col_name:\n        temp = col_name.split('[')\n        if len(temp) == 2:\n            return temp[0] + '_' + temp[1][0:-1]\n        else:\n            r = int(temp[1][0:-1])\n            c = int(temp[2][0:-1])\n            return temp[0] + '_' + str(r*10 + c)\n    else:\n        return col_name\n\nprint(rename_col(\"foo\"))\nprint(rename_col(\"foo[2]\"))\nprint(rename_col(\"foo[11][0]\"))","user":"anonymous","dateUpdated":"2020-05-02T20:55:42+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"foo\nfoo_2\nfoo_110\n"}]},"apps":[],"jobName":"paragraph_1588445456321_-1693710421","id":"20200501-225858_1540149991","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2851","dateFinished":"2020-05-02T20:55:42+0000","dateStarted":"2020-05-02T20:55:42+0000"},{"text":"%spark.pyspark\n\nold_cols = df_flat.columns\nnew_cols = []\n\nfor c in old_cols:\n    if '[' in c:\n        new_cols.append(rename_col(c))\n    else:\n        new_cols.append(c)\n\nfor i in range(len(new_cols)):\n    df_flat = df_flat.withColumnRenamed(old_cols[i], new_cols[i])\n","user":"anonymous","dateUpdated":"2020-05-02T20:55:44+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456321_-946093858","id":"20200501-142446_872410398","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2852","dateFinished":"2020-05-02T20:55:49+0000","dateStarted":"2020-05-02T20:55:44+0000"},{"text":"%spark.pyspark\nimport pyspark.sql.functions as psf\nfrom pyspark.ml.linalg import Vectors\nfrom pyspark.ml.feature import VectorAssembler\nfrom pyspark.ml.feature import MinMaxScaler\nfrom pyspark.ml import Pipeline\nfrom pyspark.sql.types import DoubleType\nfrom pyspark.sql.functions import monotonically_increasing_id \nimport pyspark\n\n# this cell serves to apply MinMaxScaler for each scalar feature\n\n# create index\ndf_scale = df_flat.withColumn(\"ID\", monotonically_increasing_id())\n\n\n# selected features\n# feature_cols = ['artist_familiarity', 'artist_hotttnesss', 'danceability', 'duration', 'end_of_fade_in', \\\n#            'energy', 'key', 'key_confidence', 'loudness', 'mode', 'mode_confidence', 'song_hotttnesss', \\\n#            'start_of_fade_out', 'tempo', 'time_signature', 'time_signature_confidence', 'year']\n\n\n# selected scalar features\nfeature_cols = ['artist_familiarity', 'danceability', 'duration', 'end_of_fade_in', \\\n                'key', 'key_confidence', 'loudness', 'mode', \\\n                'tempo', 'time_signature', 'time_signature_confidence']\n\nfor i in range(10):\n    feature_cols.append('segments_loudness_max_time_' + str(i))\n    feature_cols.append('segments_loudness_max_' + str(i))\n    feature_cols.append('segments_confidence_' + str(i))\n\nfor i in range(120):\n    feature_cols.append('segments_pitches_' + str(i))\n    feature_cols.append('segments_timbre_' + str(i))\n\nassembler = VectorAssembler(inputCols=feature_cols,outputCol=\"raw_features\").setHandleInvalid(\"skip\")\n\ndf_scale = assembler.transform(df_scale).select(\"ID\", \"song_id\", \"track_id\", \"raw_features\")\n\nscaler = MinMaxScaler(inputCol=\"raw_features\", outputCol=\"scaled_features\")\n\nscalerModel = scaler.fit(df_scale)\n\ndf_scale = scalerModel.transform(df_scale).select(\"ID\", \"song_id\", \"track_id\", \"scaled_features\").persist(pyspark.StorageLevel.DISK_ONLY)\n\n","user":"anonymous","dateUpdated":"2020-05-02T20:56:00+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456321_376981079","id":"20200502-012216_1925140839","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2853","dateFinished":"2020-05-02T20:56:48+0000","dateStarted":"2020-05-02T20:56:00+0000"},{"text":"%spark.pyspark\n# ONLY FOR TESTING PURPOSE\n#x = np.array([1,2,3])\n#y = SparseVector(5, x, [1.0 for i in range(len(x))])\n#z = y.indices +3\n#t = y.size\n#w = SparseVector(t, z, [1.0 for i in range(len(x))])\n#isinstance(w, SparseVector)","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456321_-833262461","id":"20200430-190613_1038893876","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2854"},{"text":"%spark.pyspark\nfrom pyspark.ml.linalg import Vectors, VectorUDT\nfrom pyspark.mllib.linalg import SparseVector\n\n#x = row.rdd.map(lambda x:x[0]).take(1)[0]\n#y = np.asarray(x).nonzero()[0]\n#SparseVector(262144, {i:1.0 for i in (y + offset)})\n\n#def add_offset(row, offset=17):\n#    idx = row.indices + offset\n#    sz = row.size\n#    return SparseVector(sz, idx, [1.0 for i in range(len(idx))])\n\n# ADD offset to the sparse vector indices\n\n# offset is the length of selected feature columns\n# offset = len(feature_cols)\n\n#udf_add_offset = udf(lambda row: Vectors.sparse(row.size, row.indices+offset, [1.0 for i in range(len(row.indices))]), VectorUDT())\n\n#df_scale = df_scale.withColumn(\"string_features\", udf_add_offset(\"string_features\"))\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456322_2129196488","id":"20200430-184253_1917851315","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2855"},{"text":"%spark.pyspark\nprint(df_scale.take(1))","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"[Row(ID=0, song_id='sohvdok12ab017d55f', track_id='trquuqu128f9322fcb', scaled_features=DenseVector([0.5763, 0.5, 0.0783, 0.0001, 0.1818, 0.01, 0.8405, 1.0, 0.5424, 0.5714, 0.464, 0.0003, 0.0051, 1.0, 0.0008, 0.6364, 1.0, 0.0002, 0.7397, 1.0, 0.0005, 0.6669, 1.0, 0.0004, 0.7246, 1.0, 0.0007, 0.6414, 1.0, 0.0005, 0.6612, 1.0, 0.0009, 0.6012, 1.0, 0.0002, 0.5185, 1.0, 0.0003, 0.659, 1.0, 0.2524, 0.6685, 0.4795, 0.5117, 0.4649, 0.5491, 0.4976, 0.5563, 0.3939, 0.3899, 0.4053, 0.524, 0.5147, 0.4608, 0.3935, 0.4342, 0.4287, 0.4302, 0.515, 0.3943, 0.3028, 0.4673, 0.5416, 0.3336, 0.5532, 0.2559, 0.4038, 0.274, 0.4322, 0.4611, 0.4548, 0.3974, 0.3594, 0.5768, 0.5177, 0.4052, 0.4671, 0.5811, 0.425, 0.5228, 0.302, 0.3009, 0.5321, 0.427, 0.4412, 0.3657, 0.3793, 0.4465, 0.5311, 0.415, 0.4833, 0.3877, 0.3813, 0.495, 0.4275, 0.6378, 0.3896, 0.3716, 0.5672, 0.6139, 0.2299, 0.445, 0.3938, 0.4596, 0.3957, 0.2576, 0.3572, 0.3904, 0.6029, 0.3212, 0.3953, 0.3264, 0.5364, 0.4439, 0.4849, 0.4219, 0.5252, 0.4578, 0.3491, 0.4446, 0.2021, 0.5229, 0.3183, 0.4718, 0.5438, 0.3736, 0.4879, 0.3584, 0.4308, 0.3345, 0.3425, 0.5012, 0.4217, 0.2934, 0.4637, 0.2763, 0.4911, 0.539, 0.5224, 0.3831, 0.2648, 0.5281, 0.3621, 0.466, 0.3344, 0.5093, 0.3506, 0.3374, 0.3656, 0.3415, 0.5469, 0.3523, 0.4736, 0.3997, 0.4307, 0.3866, 0.4401, 0.4985, 0.5436, 0.5416, 0.4616, 0.5096, 0.3665, 0.5233, 0.2693, 0.5095, 0.3712, 0.4073, 0.3508, 0.524, 0.466, 0.4088, 0.4265, 0.3865, 0.426, 0.4094, 0.4253, 0.3785, 0.3441, 0.4161, 0.3896, 0.5425, 0.3128, 0.4846, 0.3215, 0.4816, 0.2973, 0.3731, 0.349, 0.3118, 0.3313, 0.5281, 0.4015, 0.4684, 0.4237, 0.4495, 0.4864, 0.4255, 0.4874, 0.5988, 0.6262, 0.6177, 0.3953, 0.4914, 0.4629, 0.3902, 0.3733, 0.5166, 0.4312, 0.549, 0.4015, 0.4528, 0.4457, 0.4119, 0.5208, 0.5013, 0.4377, 0.5312, 0.5178, 0.5378, 0.4846, 0.6745, 0.381, 0.5982, 0.5445, 0.5567, 0.3946, 0.6046, 0.424, 0.5046, 0.5342, 0.3573, 0.488, 0.3913, 0.3112, 0.4919, 0.4106, 0.4134, 0.415, 0.4714, 0.5064, 0.5695, 0.4534, 0.5953, 0.394, 0.5193, 0.4574, 0.3571, 0.5001, 0.431, 0.4244, 0.5012, 0.4579, 0.502, 0.4982, 0.5475, 0.3529, 0.4043, 0.4081, 0.3658, 0.3967, 0.5839, 0.4034, 0.6076, 0.2668, 0.5037, 0.5568, 0.3908, 0.4087, 0.4054, 0.4421, 0.5205, 0.3734, 0.3769, 0.4581, 0.462, 0.495, 0.3437, 0.4411, 0.4802]))]\n"}]},"apps":[],"jobName":"paragraph_1588445456322_1400249434","id":"20200502-015958_34230006","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2856"},{"text":"%spark.pyspark\nprint(df_selected.count())","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"980320\n"}]},"apps":[],"jobName":"paragraph_1588445456322_-2139763039","id":"20200430-182355_480687666","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2858"},{"text":"%spark.pyspark\nfrom pyspark.sql.types import DoubleType\nfrom pyspark.ml.feature import Normalizer\nfrom pyspark.mllib.linalg.distributed import IndexedRow, IndexedRowMatrix\n\n\n# udf dot\ndot_udf = psf.udf(lambda x,y: float(x.dot(y)), DoubleType())\n\nnormalizer = Normalizer(inputCol=\"scaled_features\", outputCol=\"normed_features\")\ndf_normalized = normalizer.transform(df_scale)\n\nprint(df_normalized)\n\n#df_normlized.alias(\"i\").join(df_normlized.alias(\"j\"), psf.col(\"i.ID\") < psf.col(\"j.ID\"))\\\n#    .select(\n#        psf.col(\"i.ID\").alias(\"i\"), \n#        psf.col(\"j.ID\").alias(\"j\"), \n#        dot_udf(\"i.norm\", \"j.norm\").alias(\"cos\"))\\\n#    .sort(\"i\", \"j\")\\\n#    .show(10)","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"DataFrame[ID: bigint, song_id: string, track_id: string, scaled_features: vector, normed_features: vector]\n"}]},"apps":[],"jobName":"paragraph_1588445456323_835981780","id":"20200418-100325_570748945","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2859"},{"text":"%spark.pyspark\nimport numpy as np\nfrom pyspark.ml.feature import Normalizer\n\nW = np.load(\"/home/hadoop/new_fuck.npy\")\n\n\ndef compute_wfeat(row, weights):\n    return (np.array(row) * weights).tolist()\n\nudf_wfeat = udf(lambda row: compute_wfeat(row, W), ArrayType(FloatType()))\n\n# df with weighted features\ndf_wf = df_scale.withColumn(\"weighted_features\", udf_wfeat(\"scaled_features\"))\n\n# flatten array features\ndef vectorizer_1d(x):\n    return Vectors.dense(x)\n\nudf_vectorizer_1d = udf(vectorizer_1d, VectorUDT())\n\ndf_wf = df_wf.withColumn(\"weighted_features\", udf_vectorizer_1d(\"weighted_features\"))\n\nnormalizer = Normalizer(inputCol=\"weighted_features\", outputCol=\"normed_wf\")\n\ndf_normed = normalizer.transform(df_wf).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:57:00+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588449101862_-649659629","id":"20200502-195141_503537567","dateCreated":"2020-05-02T19:51:41+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7213","dateFinished":"2020-05-02T20:57:01+0000","dateStarted":"2020-05-02T20:57:00+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\nfrom pyspark.sql.functions import col, collect_list, concat_ws, udf\n\n# load training data\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\nrddTasteVisible = sc.textFile(\"file:///home/hadoop/EvalDataYear1MSDWebsite/year1_valid_triplets_visible.txt\").map(lambda x: x.split(\"\\t\")).map(lambda p: (p[0], p[1], int(p[2])))\nschemaTasteVisible = StructType([StructField(\"user\", StringType(), True), StructField(\"song\", StringType(), True), StructField(\"playCount\", IntegerType(), True)])\ndfTasteVisible = sqlContext.createDataFrame(rddTasteVisible, schema=schemaTasteVisible)\ndfTasteVisibleLimit = dfTasteVisible.withColumn(\"song\", udf_lower_sid(\"song\")).select('user', 'song').groupBy('user').agg(collect_list('song').alias('song')).orderBy('user').limit(2000).withColumn('song', explode('song')).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:57:43+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588448941002_-115781610","id":"20200502-194901_1571611511","dateCreated":"2020-05-02T19:49:01+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7069","dateFinished":"2020-05-02T20:57:32+0000","dateStarted":"2020-05-02T20:57:32+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\n# create df train\n# |song_id|nwf|\n\n# distinct songs\ndf_song_tr = dfTasteVisibleLimit.select(\"song\").distinct().persist(pyspark.StorageLevel.DISK_ONLY)\n\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\n\ndf_train = df_normed.join(df_song_tr, df_song_tr.song == df_normed.song_id).select(col(\"song_id\").alias(\"song_id_tr\"), col(\"normed_wf\").alias(\"normed_wf_tr\")).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:57:36+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588448960136_464810751","id":"20200502-194920_1092254066","dateCreated":"2020-05-02T19:49:20+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7141","dateFinished":"2020-05-02T20:57:37+0000","dateStarted":"2020-05-02T20:57:36+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\nrddTasteHidden = sc.textFile(\"file:///home/hadoop/EvalDataYear1MSDWebsite/year1_valid_triplets_hidden.txt\").map(lambda x: x.split(\"\\t\")).map(lambda p: (p[0], p[1], int(p[2])))\nschemaTasteHidden = StructType([StructField(\"user\", StringType(), True), StructField(\"song\", StringType(), True), StructField(\"playCount\", IntegerType(), True)])\ndfTasteHidden = sqlContext.createDataFrame(rddTasteHidden, schema=schemaTasteHidden)\ndfTasteHiddenLimit = dfTasteHidden.withColumn(\"song\", udf_lower_sid(\"song\")).select('user', 'song').groupBy('user').agg(collect_list('song').alias('song')).orderBy('user').limit(2000).withColumn('song', explode('song')).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:57:40+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456323_1844346546","id":"20200430-181552_1964552012","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2860","dateFinished":"2020-05-02T20:57:40+0000","dateStarted":"2020-05-02T20:57:40+0000"},{"text":"%spark.pyspark\n\n# create df_eval\n\ndf_song_ev = dfTasteHiddenLimit.select(\"song\").distinct().persist(pyspark.StorageLevel.DISK_ONLY)\n\n#df_song_ev = df_song_ev.withColumn(\"song\", udf_lower_sid(\"song\")).persist(pyspark.StorageLevel.DISK_ONLY)\n\ndf_eval = df_normed.join(df_song_ev, df_song_ev.song == df_normed.song_id).select(col(\"song_id\").alias(\"song_id_ev\"), col(\"normed_wf\").alias(\"normed_wf_ev\")).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:57:43+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588449972192_-1568037999","id":"20200502-200612_2136175406","dateCreated":"2020-05-02T20:06:12+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7528","dateFinished":"2020-05-02T20:57:43+0000","dateStarted":"2020-05-02T20:57:43+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\nfrom pyspark.sql.functions import monotonically_increasing_id, col\n\ndot_udf = udf(lambda x,y: float(x.dot(y)), FloatType())\n\ndf_res = df_train.crossJoin(df_eval).withColumn(\"cos_sim\", dot_udf(\"normed_wf_tr\", \"normed_wf_ev\")).select(\"song_id_tr\", \"song_id_ev\", \"cos_sim\").persist(pyspark.StorageLevel.DISK_ONLY)\n\n\n","user":"anonymous","dateUpdated":"2020-05-02T20:57:46+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588450114937_1903470594","id":"20200502-200834_420276293","dateCreated":"2020-05-02T20:08:34+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7651","dateFinished":"2020-05-02T20:57:46+0000","dateStarted":"2020-05-02T20:57:46+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\ndf_res.count()","user":"anonymous","dateUpdated":"2020-05-02T20:57:49+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588451624737_475545920","id":"20200502-203344_541623024","dateCreated":"2020-05-02T20:33:44+0000","status":"RUNNING","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8900","dateFinished":"2020-05-02T20:52:00+0000","dateStarted":"2020-05-02T20:57:49+0000","errorMessage":""},{"text":"%spark.pyspark\ndf_res.show(10)","user":"anonymous","dateUpdated":"2020-05-02T20:32:25+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588450739519_-1412903768","id":"20200502-201859_1575304679","dateCreated":"2020-05-02T20:18:59+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8042","dateFinished":"2020-05-02T20:32:37+0000","dateStarted":"2020-05-02T20:32:25+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------+------------------+----------+\n|        song_id_tr|        song_id_ev|   cos_sim|\n+------------------+------------------+----------+\n|soaejls12a8ae47add|soajnft12ab017e02f| 0.9965231|\n|soaejls12a8ae47add|soajsca12a58a7ad0e|0.98905724|\n|soaejls12a8ae47add|soakxwv12a6d4f415a| 0.8806414|\n|soaejls12a8ae47add|soasnqv12a8ae48819|0.95549244|\n|soaejls12a8ae47add|soaxhhp12a6d4fccda|0.47418576|\n|soaejls12a8ae47add|sobarip12a8c136750| 0.4742662|\n|soaejls12a8ae47add|sobcmfz12ab017f89b| 0.8806692|\n|soaejls12a8ae47add|sobgppp12ab017d4ac|0.47359744|\n|soaejls12a8ae47add|sobhgsi12a6d4f7b31| 0.9748952|\n|soaejls12a8ae47add|sobihcz12a6d4f8549|0.47414964|\n+------------------+------------------+----------+\nonly showing top 10 rows\n\n"}]}},{"text":"%spark.pyspark\ndf_distinct_user_id_hidden = dfTasteHidden.select(\"user\").distinct()\n\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\n\ndf_distinct_user_id_hidden = df_distinct_user_id_hidden.withColumn(\"user\", udf_lower_sid(\"user\")).persist(pyspark.StorageLevel.DISK_ONLY)\n\ndf_distinct_user_id_hidden.count()\n","user":"anonymous","dateUpdated":"2020-05-02T19:23:51+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588447239751_1098691472","id":"20200502-192039_663232252","dateCreated":"2020-05-02T19:20:39+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6606","dateFinished":"2020-05-02T19:23:53+0000","dateStarted":"2020-05-02T19:23:51+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"10000\n"}]}},{"text":"%spark.pyspark\ndf_distinct_user_id_visible = dfTasteVisible.select(\"user\").distinct()\n\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\n\ndf_distinct_user_id_visible = df_distinct_user_id_hidden.withColumn(\"user\", udf_lower_sid(\"user\")).persist(pyspark.StorageLevel.DISK_ONLY)\n\ndf_distinct_user_id_visible.count()","user":"anonymous","dateUpdated":"2020-05-02T19:25:03+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588447408425_-234339756","id":"20200502-192328_174147878","dateCreated":"2020-05-02T19:23:28+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6716","dateFinished":"2020-05-02T19:25:03+0000","dateStarted":"2020-05-02T19:25:02+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"10000\n"}]}},{"text":"%spark.pyspark\ndf_distinct_user_id_visible.join(df_distinct_user_id_hidden, df_distinct_user_id_visible.user == df_distinct_user_id_hidden.user).count()","user":"anonymous","dateUpdated":"2020-05-02T19:25:56+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588447505624_-1583207859","id":"20200502-192505_1109199098","dateCreated":"2020-05-02T19:25:05+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6954","dateFinished":"2020-05-02T19:25:57+0000","dateStarted":"2020-05-02T19:25:56+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"10000\n"}]}},{"text":"%spark.pyspark\ndf_distinct_user_id_hidden = dfTasteHidden.select(\"user\").distinct()\n\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\n\ndf_distinct_user_id_hidden = df_distinct_user_id_hidden.withColumn(\"user\", udf_lower_sid(\"user\")).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T19:56:52+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588447417207_1515978497","id":"20200502-192337_1572688473","dateCreated":"2020-05-02T19:23:37+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6815","dateFinished":"2020-05-02T19:56:52+0000","dateStarted":"2020-05-02T19:56:52+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\nfrom pyspark.sql.types import StringType\nfrom pyspark.sql.functions import col, collect_list, concat_ws, udf\n\ndef myFunc(data_list):\n    return data_list[0]\n\nmyUdf = udf(myFunc, StringType())\n\n# df.withColumn('data', concat_ws(',', col('B'), col('C'))) \\\n#   .groupBy('A').agg(collect_list('data').alias('data'))\n#   .withColumn('data', myUdf('data'))\n  \n\ndf_user_to_TS_limit = dfTasteVisibleLimit.select('user', 'song').groupBy('user').agg(collect_list('song').alias('song')).withColumn('song_id_tr', myUdf('song')).drop('song').persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:46:46+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588449414772_171287028","id":"20200502-195654_1301913299","dateCreated":"2020-05-02T19:56:54+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7375","dateFinished":"2020-05-02T20:28:09+0000","dateStarted":"2020-05-02T20:28:09+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\ndf_user_to_TS.show(10)","user":"anonymous","dateUpdated":"2020-05-02T20:28:10+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588450782237_-910842276","id":"20200502-201942_1133028550","dateCreated":"2020-05-02T20:19:42+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8146","dateFinished":"2020-05-02T20:28:11+0000","dateStarted":"2020-05-02T20:28:10+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+------------------+\n|                user|        song_id_tr|\n+--------------------+------------------+\n|022ef17932f3a59b1...|SOVIPXF12AC3DF82FB|\n|027c302acd37d10bf...|SOLNJUS12A6D4F8BB1|\n|05114c576afe79bc0...|SOOGJXW12A6D4FBBCC|\n|1769dfb6c39475eba...|SOZFKBR12A6D4F6CC0|\n|1bc9058b068ba521d...|SONHVVE12AB018D038|\n|1bf41057e08750219...|SORLZWT12AC468B051|\n|1ef2b7de56b01bd0d...|SOSBJSU12A8C138469|\n|277192b5689f8a7a9...|SOFHZGL12A8C142521|\n|39148bbfce6dd2f01...|SOKKKZW12A6D4F7F88|\n|3a660e789174c404d...|SONWRWA12A6D4FD60E|\n+--------------------+------------------+\nonly showing top 10 rows\n\n"}]}},{"text":"%spark.pyspark\ndf_user_cos = df_res.join(df_user_to_TS_limit, df_res.song_id_tr = df_user_to_TS_limit.song_id_tr).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:46:51+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588451245055_-832459814","id":"20200502-202725_1420817126","dateCreated":"2020-05-02T20:27:25+0000","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8603"},{"text":"%spark.pyspark\ndf_user_to_FSs = dfTasteHidden.select('user', 'song').groupBy('user').agg(collect_list('song').alias('FSs')).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:26:38+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588450917011_-1834987953","id":"20200502-202157_61486339","dateCreated":"2020-05-02T20:21:57+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8367","dateFinished":"2020-05-02T20:26:38+0000","dateStarted":"2020-05-02T20:26:38+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\ndf_user_to_FSs_limit = dfTasteHiddenLimit.select('user', 'song').groupBy('user').agg(collect_list('song').alias('FSs')).persist(pyspark.StorageLevel.DISK_ONLY)","user":"anonymous","dateUpdated":"2020-05-02T20:47:52+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588451709545_381960524","id":"20200502-203509_1401671780","dateCreated":"2020-05-02T20:35:09+0000","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8981"},{"text":"%spark.pyspark\ndf_user_to_FSs.show(10)","user":"anonymous","dateUpdated":"2020-05-02T20:26:40+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588451101773_-1484631549","id":"20200502-202501_2067783162","dateCreated":"2020-05-02T20:25:01+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:8474","dateFinished":"2020-05-02T20:26:40+0000","dateStarted":"2020-05-02T20:26:40+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+--------------------+\n|                user|                 FSs|\n+--------------------+--------------------+\n|87ff12276e115e2d8...|[SOSNMJN12A58A7A6...|\n|8b8119e53dbe3a54d...|[SOXPYFP12A6701C3...|\n|8cb2b2dabde47a4a5...|[SOEIJIX12A6D4FAF...|\n|8ddca6bbce8bbdbc4...|[SOUYULH12A8C141F...|\n|8fbcfc926f611485e...|[SOBONKR12A58A7A7...|\n|92df21b2d860a5e61...|[SOVMEMU12AB0183A...|\n|96bec494319aeb619...|[SOBRZTW12AB0182B...|\n|a73319a1abe5a1271...|[SOHOZXZ12AC46878...|\n|aa7245396a0076a46...|[SOKUMIK12A6D4F5F...|\n|b2aae730c0dccaf80...|[SOBPFAB12A8C140A...|\n+--------------------+--------------------+\nonly showing top 10 rows\n\n"}]}},{"text":"%spark.pyspark\ndfTasteHidden.take(10)","user":"anonymous","dateUpdated":"2020-05-02T20:14:37+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588449398356_1833120353","id":"20200502-195638_1266477122","dateCreated":"2020-05-02T19:56:38+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7294","dateFinished":"2020-05-02T20:14:26+0000","dateStarted":"2020-05-02T20:14:26+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"[Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOXHPVI12A6D4F903A', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOKGAEC12A6310D847', playCount=2), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SODQIVP12AB017B16F', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOLLNTU12A6701CFDC', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOSWVEZ12A6D4F941E', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOYQKHX12A6BD4F039', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOYNJCS12A67ADE35F', playCount=3), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOGMOQG12AB017C235', playCount=2), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOMFCUM12A67FFB35C', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOUHFOP12AB0180942', playCount=4)]\n"}]}},{"text":"%spark.pyspark\n\nfrom pyspark.sql.functions import monotonically_increasing_id, col\n\n# create index\ndf_joined = df_joined.withColumn(\"ID\", monotonically_increasing_id()).persist(pyspark.StorageLevel.DISK_ONLY)\n\ndot_udf = udf(lambda x,y: float(x.dot(y)), FloatType())\n\n\n#df_res = df_joined.alias(\"i\").join(df_joined.alias(\"j\"), col(\"i.ID\") < col(\"j.ID\"))\\\n#    .select(\n#        col(\"i.ID\").alias(\"i\"), \n#        col(\"j.ID\").alias(\"j\"), \n#        col(\"i.song_id\").alias(\"song_id_1\"),\n#        col(\"j.song_id\").alias(\"song_id_2\"),\n#        dot_udf(\"i.normed_wf\", \"j.normed_wf\").alias(\"cos_sim\"))\\\n#    .sort(\"i\", \"j\") ","user":"anonymous","dateUpdated":"2020-05-02T19:32:23+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588446711643_-1491584160","id":"20200502-191151_544933350","dateCreated":"2020-05-02T19:11:51+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6361","dateFinished":"2020-05-02T19:20:31+0000","dateStarted":"2020-05-02T19:20:30+0000","results":{"code":"SUCCESS","msg":[]}},{"text":"%spark.pyspark\ndf_res.show(10)","user":"anonymous","dateUpdated":"2020-05-02T19:20:37+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588446916839_67301637","id":"20200502-191516_646009864","dateCreated":"2020-05-02T19:15:16+0000","status":"ABORT","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6474","dateFinished":"2020-05-02T19:21:46+0000","dateStarted":"2020-05-02T19:20:37+0000","results":{"code":"ERROR","msg":[{"type":"TEXT","data":"Fail to execute line 1: df_res.show(10)\nTraceback (most recent call last):\n  File \"/tmp/zeppelin_pyspark-9020773312073553344.py\", line 380, in <module>\n    exec(code, _zcUserQueryNameSpace)\n  File \"<stdin>\", line 1, in <module>\n  File \"/usr/lib/spark/python/lib/pyspark.zip/pyspark/sql/dataframe.py\", line 380, in show\n    print(self._jdf.showString(n, 20, vertical))\n  File \"/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py\", line 1255, in __call__\n    answer = self.gateway_client.send_command(command)\n  File \"/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py\", line 985, in send_command\n    response = connection.send_command(command)\n  File \"/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py\", line 1152, in send_command\n    answer = smart_decode(self.stream.readline()[:-1])\n  File \"/usr/lib64/python2.7/socket.py\", line 451, in readline\n    data = self._sock.recv(self._rbufsize)\n  File \"/usr/lib/spark/python/lib/pyspark.zip/pyspark/context.py\", line 278, in signal_handler\n    raise KeyboardInterrupt()\nKeyboardInterrupt\n"}]}},{"text":"%spark.pyspark\ndf_joined.show(10)","user":"anonymous","dateUpdated":"2020-05-02T19:10:21+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588446500083_-1318337815","id":"20200502-190820_1144641875","dateCreated":"2020-05-02T19:08:20+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6240","dateFinished":"2020-05-02T19:11:21+0000","dateStarted":"2020-05-02T19:10:21+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------+--------------------+\n|           song_id|           normed_wf|\n+------------------+--------------------+\n|soajnft12ab017e02f|[0.00316970058078...|\n|soajsca12a58a7ad0e|[0.00332508389232...|\n|soakxwv12a6d4f415a|[0.00357136069109...|\n|soasnqv12a8ae48819|[0.00265415710017...|\n|soaxhhp12a6d4fccda|[0.00663494067182...|\n|sobarip12a8c136750|[0.00613893071796...|\n|sobcmfz12ab017f89b|[0.00246598710704...|\n|sobgppp12ab017d4ac|[0.01063522238781...|\n|sobhgsi12a6d4f7b31|[0.00238766492958...|\n|sobihcz12a6d4f8549|[0.00374552249215...|\n+------------------+--------------------+\nonly showing top 10 rows\n\n"}]}},{"text":"%spark.pyspark\ndf_distinct_song.show(10)","user":"anonymous","dateUpdated":"2020-05-02T19:00:29+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588445906470_58954410","id":"20200502-185826_491310948","dateCreated":"2020-05-02T18:58:26+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:5913","dateFinished":"2020-05-02T19:00:30+0000","dateStarted":"2020-05-02T19:00:29+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------+\n|              song|\n+------------------+\n|SONGYMA12A6D4F8A32|\n|SOTMLOE12A8C133F15|\n|SOQQGPO12AB0181F82|\n|SOEZUSU12A8AE480B1|\n|SOLIKOY12AB0186313|\n|SOUQOMC12A8C135F64|\n|SOGAPRD12A8C13C4A7|\n|SOAHSJC12AB018BBB2|\n|SOGXQYC12AB0183AE5|\n|SOLFSBW12AB017F2FE|\n+------------------+\nonly showing top 10 rows\n\n"}]}},{"text":"%spark.pyspark\nrddTasteVisible = sc.textFile(\"file:///home/hadoop/EvalDataYear1MSDWebsite/year1_valid_triplets_visible.txt\").map(lambda x: x.split(\"\\t\")).map(lambda p: (p[0], p[1], int(p[2])))\nschemaTasteVisible = StructType([StructField(\"user\", StringType(), True), StructField(\"song\", StringType(), True), StructField(\"playCount\", IntegerType(), True)])\ndfTasteVisible = sqlContext.createDataFrame(rddTasteVisible, schema=schemaTasteHidden)\n","user":"anonymous","dateUpdated":"2020-05-02T19:24:55+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456324_-41728538","id":"20200502-023038_546515835","dateCreated":"2020-05-02T18:50:56+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:2862","dateFinished":"2020-05-02T19:24:55+0000","dateStarted":"2020-05-02T19:24:55+0000"},{"text":"%spark.pyspark\ndfTasteHidden.take(10)","user":"anonymous","dateUpdated":"2020-05-02T19:00:00+0000","config":{"colWidth":12,"fontSize":9,"enabled":true,"results":{},"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"editorMode":"ace/mode/python"},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588445971710_-1724462402","id":"20200502-185931_247035839","dateCreated":"2020-05-02T18:59:31+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:6001","dateFinished":"2020-05-02T18:59:56+0000","dateStarted":"2020-05-02T18:59:49+0000","results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"[Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOXHPVI12A6D4F903A', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOKGAEC12A6310D847', playCount=2), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SODQIVP12AB017B16F', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOLLNTU12A6701CFDC', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOSWVEZ12A6D4F941E', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOYQKHX12A6BD4F039', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOYNJCS12A67ADE35F', playCount=3), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOGMOQG12AB017C235', playCount=2), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOMFCUM12A67FFB35C', playCount=1), Row(user=u'0007140a3796e901f3190f12e9de6d7548d4ac4a', song=u'SOUHFOP12AB0180942', playCount=4)]\n"}]}},{"text":"%spark.pyspark\nfrom pyspark.sql import Window\nimport pyspark.sql.functions as f\n\nw = Window.partitionBy('user')\ndf_user = dfTasteVisible.select('user', 'song', f.count('user').over(w).alias('n')).sort('user', 'song')\n\ndef filter_gt1(row):\n    return row > 1\n\nudf_filter_gt1 = udf(lambda row: filter_gt1(row), BooleanType())\nudf_lower_sid = udf(lambda row: row.lower(), StringType())\n\ndf_user = df_user.filter(udf_filter_gt1(\"n\"))\ndf_user = df_user.withColumn(\"song\", udf_lower_sid(\"song\"))\n\ndf_train = df_normalized.join(df_user, df_user.song == df_normalized.song_id).select(\"song_id\", \"scaled_features\", \"user\")\n\n\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456324_1762079786","id":"20200502-022842_432906607","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2863"},{"text":"%spark.pyspark\nto_array = udf(lambda v: v.toArray().tolist(), ArrayType(FloatType()))\n\ndf_train = df_train.withColumn('scaled_features', to_array('scaled_features'))\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456325_-613639240","id":"20200502-040522_1838165061","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2864"},{"text":"%spark.pyspark\n#import pandas as pd\n#df_sub = df_train.limit(10000)\ndf_train.write.mode(\"overwrite\").parquet(\"s3a://msdbucket/features_full_train\")\n#df_sub.toPandas().to_pickle(\"./dummy.pkl\")\n#df_train.count()\n\n#df_train.toPandas().to_pickle(\"./full.pkl\")\n\n\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456325_1448519544","id":"20200502-032450_1412284508","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2865"},{"text":"%spark.pyspark\nimport numpy as np\n\nW = np.load(\"/home/hadoop/new_fuck.npy\")","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456325_1555501027","id":"20200502-051750_361728905","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2866"},{"text":"%spark.pyspark\ndef compute_wfeat(row, weights):\n    return (np.array(row) * weights).tolist()\n\nudf_wfeat = udf(lambda row: compute_wfeat(row, W), ArrayType(FloatType()))\n# df with weighted features\ndf_wf = df_scale.withColumn(\"weighted_features\", udf_wfeat(\"scaled_features\"))\n\n# flatten array features\ndef vectorizer_1d(x):\n    return Vectors.dense(x)\n\nudf_vectorizer_1d = udf(vectorizer_1d, VectorUDT())\n\ndf_wf = df_wf.withColumn(\"weighted_features\", udf_vectorizer_1d(\"weighted_features\"))\n\nnormalizer = Normalizer(inputCol=\"weighted_features\", outputCol=\"normed_wf\")\n\ndf_normalized = normalizer.transform(df_wf)\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456326_83259531","id":"20200502-052519_913215292","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2867"},{"text":"%spark.pyspark\n# compute similarity\n\ninput_song_id = \"soaejls12a8ae47add\"\n\nsong_wf = df_normalized.filter(df_normalized.song_id==input_song_id).select(\"normed_wf\").take(1)[0]\n\ndot_udf = udf(lambda x: float(np.dot(np.array(x), np.array(song_wf)[0])), DoubleType())\n\ndf_res = df_normalized.withColumn(\"cos_sim\", dot_udf(\"normed_wf\")).sort(\"cos_sim\", ascending=False).limit(11)\n\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[]},"apps":[],"jobName":"paragraph_1588445456326_-287859648","id":"20200502-053724_227932415","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2868"},{"text":"%spark.pyspark\ndf_res.select(\"song_id\", \"cos_sim\").show()","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------+------------------+\n|           song_id|           cos_sim|\n+------------------+------------------+\n|soaejls12a8ae47add|0.9999999999999987|\n|sozqvcv12a58839329|0.9999955901751498|\n|sohbuvv12aaf3b34c3|0.9999954931442198|\n|sonitfv12a6d4f97bf|0.9999953374342567|\n|sojhltb12af72a78c9|0.9999952489939011|\n|sovepcs12a8c13a04a| 0.999995128116204|\n|sobebcp12ab0182aee|0.9999951121464633|\n|sotiwil12a8c137e70| 0.999994825873491|\n|sopbfcz12a8c13ede2|0.9999946809680348|\n|sozsjbh12a8c14355d|0.9999946429838474|\n|somchqr12a6d4f74ef|0.9999946083648367|\n+------------------+------------------+\n\n"}]},"apps":[],"jobName":"paragraph_1588445456326_1407611804","id":"20200502-054120_479119693","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2869"},{"text":"%spark.pyspark\ndf_normalized.show(2)","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+------------------+--------------------+--------------------+--------------------+--------------------+\n|           song_id|     scaled_features|                user|   weighted_features|           normed_wf|\n+------------------+--------------------+--------------------+--------------------+--------------------+\n|soaejls12a8ae47add|[0.8036843, 0.5, ...|75a2408929e5a957d...|[0.55277822094523...|[0.00419088497799...|\n|soajnft12ab017e02f|[0.638614, 0.5, 0...|42810a06cb21fa4ff...|[0.43924202996811...|[0.00316970058078...|\n+------------------+--------------------+--------------------+--------------------+--------------------+\nonly showing top 2 rows\n\n"}]},"apps":[],"jobName":"paragraph_1588445456327_801601590","id":"20200502-053229_1035045143","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2870"},{"text":"%spark.pyspark\nimport os\nos.getcwd()","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"'/mnt/var/lib/zeppelin'\n"}]},"apps":[],"jobName":"paragraph_1588445456327_459061988","id":"20200502-035053_277952523","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2871"},{"text":"%spark.pyspark\nprint(df_train)\ndf_normalized.select(\"song_id\").show(3)\ndf_user.select(\"song\").show(3)","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"DataFrame[ID: bigint, song_id: string, track_id: string, features: vector, norm_feat: vector, user: string, song: string, n: bigint]\n+------------------+\n|           song_id|\n+------------------+\n|sohvdok12ab017d55f|\n|sozaxqf12a6d4fd4fc|\n|sobckyz12ab0187fb5|\n+------------------+\nonly showing top 3 rows\n\n+------------------+\n|              song|\n+------------------+\n|SOBZNPI12AF72AAFF6|\n|SOEBOWM12AB017F279|\n|SOEYJDP12A8C138D4B|\n+------------------+\nonly showing top 3 rows\n\n"}]},"apps":[],"jobName":"paragraph_1588445456327_-179460892","id":"20200502-032104_1418942219","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2872"},{"text":"%spark.pyspark\nvector_udf = F.UserDefinedFunction(lambda vector: vector.toArray().tolist(),ArrayType(DoubleType()))\ncolvalues = featurized.select(vector_udf('features').alias('features')).show()","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"+--------------------+\n|            features|\n+--------------------+\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n|[0.0, 0.0, 0.0, 0...|\n+--------------------+\nonly showing top 20 rows\n\n"}]},"apps":[],"jobName":"paragraph_1588445456328_-1055044088","id":"20200418-204228_903129046","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2873"},{"text":"%spark.pyspark\nx = featurized.select('string_features').rdd.map(lambda x:x[0]).take(1)[0]\ny = np.asarray(x).nonzero()[0]\nSparseVector(262144, {i:1.0 for i in (y + 17)})","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"results":{"code":"SUCCESS","msg":[{"type":"TEXT","data":"SparseVector(262144, {14048: 1.0, 43637: 1.0, 65781: 1.0, 105806: 1.0, 163777: 1.0, 176097: 1.0, 202469: 1.0})\n"}]},"apps":[],"jobName":"paragraph_1588445456328_290920471","id":"20200430-183158_1205498344","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2874"},{"text":"%spark.pyspark\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588445456328_606210905","id":"20200430-183158_1697375522","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2875"},{"text":"%spark.pyspark\n","user":"anonymous","dateUpdated":"2020-05-02T18:50:56+0000","config":{"editorSetting":{"language":"python","editOnDblClick":false,"completionKey":"TAB","completionSupport":true},"colWidth":12,"editorMode":"ace/mode/python","fontSize":9,"results":{},"enabled":true},"settings":{"params":{},"forms":{}},"apps":[],"jobName":"paragraph_1588445456329_938637763","id":"20200418-213523_1826916850","dateCreated":"2020-05-02T18:50:56+0000","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:2876"}],"name":"Test","id":"2F7CHX252","noteParams":{},"noteForms":{},"angularObjects":{"spark:shared_process":[]},"config":{"isZeppelinNotebookCronEnable":false,"looknfeel":"default","personalizedMode":"false"},"info":{}}