cigna1-rdd.ipynb

{"cells":[{"cell_type":"code","source":["titanic_data = spark.read.csv('/FileStore/tables/titanic_train-ac800.csv', header=True, inferSchema=True)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":1},{"cell_type":"code","source":["titanic_data.dtypes"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">2</span><span class=\"ansired\">]: </span>\n[(&apos;PassengerId&apos;, &apos;int&apos;),\n (&apos;Survived&apos;, &apos;int&apos;),\n (&apos;Pclass&apos;, &apos;int&apos;),\n (&apos;Name&apos;, &apos;string&apos;),\n (&apos;Sex&apos;, &apos;string&apos;),\n (&apos;Age&apos;, &apos;double&apos;),\n (&apos;SibSp&apos;, &apos;int&apos;),\n (&apos;Parch&apos;, &apos;int&apos;),\n (&apos;Ticket&apos;, &apos;string&apos;),\n (&apos;Fare&apos;, &apos;double&apos;),\n (&apos;Cabin&apos;, &apos;string&apos;),\n (&apos;Embarked&apos;, &apos;string&apos;)]\n</div>"]}}],"execution_count":2},{"cell_type":"code","source":["df = spark.range(10)\ndf = df.withColumn(\"B\",lit(5))\ndf2 = spark.range(10)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":3},{"cell_type":"code","source":["from pyspark.sql.functions import lit\ndf.withColumn('A', df2.id+1).count()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">AnalysisException</span>                         Traceback (most recent call last)\n<span class=\"ansigreen\">&lt;command-460960499044738&gt;</span> in <span class=\"ansicyan\">&lt;module&gt;</span><span class=\"ansiblue\">()</span>\n<span class=\"ansigreen\">      1</span> <span class=\"ansigreen\">from</span> pyspark<span class=\"ansiyellow\">.</span>sql<span class=\"ansiyellow\">.</span>functions <span class=\"ansigreen\">import</span> lit<span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">----&gt; 2</span><span class=\"ansiyellow\"> </span>df<span class=\"ansiyellow\">.</span>withColumn<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;A&apos;</span><span class=\"ansiyellow\">,</span> df2<span class=\"ansiyellow\">.</span>id<span class=\"ansiyellow\">+</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">.</span>count<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/dataframe.py</span> in <span class=\"ansicyan\">withColumn</span><span class=\"ansiblue\">(self, colName, col)</span>\n<span class=\"ansigreen\">   2012</span>         &quot;&quot;&quot;\n<span class=\"ansigreen\">   2013</span>         <span class=\"ansigreen\">assert</span> isinstance<span class=\"ansiyellow\">(</span>col<span class=\"ansiyellow\">,</span> Column<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">,</span> <span class=\"ansiblue\">&quot;col should be Column&quot;</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">-&gt; 2014</span><span class=\"ansiyellow\">         </span><span class=\"ansigreen\">return</span> DataFrame<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">.</span>_jdf<span class=\"ansiyellow\">.</span>withColumn<span class=\"ansiyellow\">(</span>colName<span class=\"ansiyellow\">,</span> col<span class=\"ansiyellow\">.</span>_jc<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">,</span> self<span class=\"ansiyellow\">.</span>sql_ctx<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   2015</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   2016</span>     <span class=\"ansiyellow\">@</span>ignore_unicode_prefix<span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py</span> in <span class=\"ansicyan\">__call__</span><span class=\"ansiblue\">(self, *args)</span>\n<span class=\"ansigreen\">   1255</span>         answer <span class=\"ansiyellow\">=</span> self<span class=\"ansiyellow\">.</span>gateway_client<span class=\"ansiyellow\">.</span>send_command<span class=\"ansiyellow\">(</span>command<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1256</span>         return_value = get_return_value(\n<span class=\"ansigreen\">-&gt; 1257</span><span class=\"ansiyellow\">             answer, self.gateway_client, self.target_id, self.name)\n</span><span class=\"ansigreen\">   1258</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1259</span>         <span class=\"ansigreen\">for</span> temp_arg <span class=\"ansigreen\">in</span> temp_args<span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/utils.py</span> in <span class=\"ansicyan\">deco</span><span class=\"ansiblue\">(*a, **kw)</span>\n<span class=\"ansigreen\">     67</span>                                              e.java_exception.getStackTrace()))\n<span class=\"ansigreen\">     68</span>             <span class=\"ansigreen\">if</span> s<span class=\"ansiyellow\">.</span>startswith<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;org.apache.spark.sql.AnalysisException: &apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 69</span><span class=\"ansiyellow\">                 </span><span class=\"ansigreen\">raise</span> AnalysisException<span class=\"ansiyellow\">(</span>s<span class=\"ansiyellow\">.</span>split<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;: &apos;</span><span class=\"ansiyellow\">,</span> <span class=\"ansicyan\">1</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">[</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">]</span><span class=\"ansiyellow\">,</span> stackTrace<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     70</span>             <span class=\"ansigreen\">if</span> s<span class=\"ansiyellow\">.</span>startswith<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;org.apache.spark.sql.catalyst.analysis&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     71</span>                 <span class=\"ansigreen\">raise</span> AnalysisException<span class=\"ansiyellow\">(</span>s<span class=\"ansiyellow\">.</span>split<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;: &apos;</span><span class=\"ansiyellow\">,</span> <span class=\"ansicyan\">1</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">[</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">]</span><span class=\"ansiyellow\">,</span> stackTrace<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">AnalysisException</span>: &apos;Resolved attribute(s) id#1553L missing from id#1548L,B#1550 in operator !Project [id#1548L, B#1550, (id#1553L + cast(1 as bigint)) AS A#1555L]. Attribute(s) with the same name appear in the operation: id. Please check if the right attribute(s) are used.;;\\n!Project [id#1548L, B#1550, (id#1553L + cast(1 as bigint)) AS A#1555L]\\n+- Project [id#1548L, 5 AS B#1550]\\n   +- Range (0, 10, step=1, splits=Some(8))\\n&apos;</div>"]}}],"execution_count":4},{"cell_type":"code","source":["df2.id"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">16</span><span class=\"ansired\">]: </span>Column&lt;b&apos;id&apos;&gt;\n</div>"]}}],"execution_count":5},{"cell_type":"code","source":["df = spark.createDataFrame([(1, 4), (2, 5), (3, 6)], [\"A\", \"B\"])"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":6},{"cell_type":"code","source":["df = df.withColumn('C',df.A + df.B)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":7},{"cell_type":"code","source":["df = df.select('*',(df.A > 2).alias(\"State\"),(lit(2).alias(\"StateB\")))"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":8},{"cell_type":"code","source":["df.show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+---+---+---+-----+------+\n  A|  B|  C|State|StateB|\n+---+---+---+-----+------+\n  1|  4|  5|false|     2|\n  2|  5|  7|false|     2|\n  3|  6|  9| true|     2|\n+---+---+---+-----+------+\n\n</div>"]}}],"execution_count":9},{"cell_type":"code","source":["import pyspark.sql.functions as F"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":10},{"cell_type":"code","source":["df.select(\"*\", F.when(df.C == 5,'Great').when(df.C == 7,'good').otherwise('Awesome').alias(\"result\")).show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+---+---+---+-----+------+-------+\n  A|  B|  C|State|StateB| result|\n+---+---+---+-----+------+-------+\n  1|  4|  5|false|     2|  Great|\n  2|  5|  7|false|     2|   good|\n  3|  6|  9| true|     2|Awesome|\n+---+---+---+-----+------+-------+\n\n</div>"]}}],"execution_count":11},{"cell_type":"code","source":["titanic_data.createOrReplaceTempView('titanic')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":12},{"cell_type":"code","source":["display(spark.sql(\"select * from titanic where Sex='female'\"))"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .table-result-container {\n    max-height: 300px;\n    overflow: auto;\n  }\n  table, th, td {\n    border: 1px solid black;\n    border-collapse: collapse;\n  }\n  th, td {\n    padding: 5px;\n  }\n  th {\n    text-align: left;\n  }\n</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>PassengerId</th><th>Survived</th><th>Pclass</th><th>Name</th><th>Sex</th><th>Age</th><th>SibSp</th><th>Parch</th><th>Ticket</th><th>Fare</th><th>Cabin</th><th>Embarked</th></tr></thead><tbody><tr><td>2</td><td>1</td><td>1</td><td>Cumings, Mrs. John Bradley (Florence Briggs Thayer)</td><td>female</td><td>38.0</td><td>1</td><td>0</td><td>PC 17599</td><td>71.2833</td><td>C85</td><td>C</td></tr><tr><td>3</td><td>1</td><td>3</td><td>Heikkinen, Miss. Laina</td><td>female</td><td>26.0</td><td>0</td><td>0</td><td>STON/O2. 3101282</td><td>7.925</td><td>null</td><td>S</td></tr><tr><td>4</td><td>1</td><td>1</td><td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td><td>female</td><td>35.0</td><td>1</td><td>0</td><td>113803</td><td>53.1</td><td>C123</td><td>S</td></tr><tr><td>9</td><td>1</td><td>3</td><td>Johnson, Mrs. Oscar W (Elisabeth Vilhelmina Berg)</td><td>female</td><td>27.0</td><td>0</td><td>2</td><td>347742</td><td>11.1333</td><td>null</td><td>S</td></tr><tr><td>10</td><td>1</td><td>2</td><td>Nasser, Mrs. Nicholas (Adele Achem)</td><td>female</td><td>14.0</td><td>1</td><td>0</td><td>237736</td><td>30.0708</td><td>null</td><td>C</td></tr><tr><td>11</td><td>1</td><td>3</td><td>Sandstrom, Miss. Marguerite Rut</td><td>female</td><td>4.0</td><td>1</td><td>1</td><td>PP 9549</td><td>16.7</td><td>G6</td><td>S</td></tr><tr><td>12</td><td>1</td><td>1</td><td>Bonnell, Miss. Elizabeth</td><td>female</td><td>58.0</td><td>0</td><td>0</td><td>113783</td><td>26.55</td><td>C103</td><td>S</td></tr><tr><td>15</td><td>0</td><td>3</td><td>Vestrom, Miss. Hulda Amanda Adolfina</td><td>female</td><td>14.0</td><td>0</td><td>0</td><td>350406</td><td>7.8542</td><td>null</td><td>S</td></tr><tr><td>16</td><td>1</td><td>2</td><td>Hewlett, Mrs. (Mary D Kingcome) </td><td>female</td><td>55.0</td><td>0</td><td>0</td><td>248706</td><td>16.0</td><td>null</td><td>S</td></tr><tr><td>19</td><td>0</td><td>3</td><td>Vander Planke, Mrs. Julius (Emelia Maria Vandemoortele)</td><td>female</td><td>31.0</td><td>1</td><td>0</td><td>345763</td><td>18.0</td><td>null</td><td>S</td></tr><tr><td>20</td><td>1</td><td>3</td><td>Masselmani, Mrs. Fatima</td><td>female</td><td>null</td><td>0</td><td>0</td><td>2649</td><td>7.225</td><td>null</td><td>C</td></tr><tr><td>23</td><td>1</td><td>3</td><td>\"McGowan, Miss. Anna \"\"Annie\"\"\"</td><td>female</td><td>15.0</td><td>0</td><td>0</td><td>330923</td><td>8.0292</td><td>null</td><td>Q</td></tr><tr><td>25</td><td>0</td><td>3</td><td>Palsson, Miss. Torborg Danira</td><td>female</td><td>8.0</td><td>3</td><td>1</td><td>349909</td><td>21.075</td><td>null</td><td>S</td></tr><tr><td>26</td><td>1</td><td>3</td><td>Asplund, Mrs. Carl Oscar (Selma Augusta Emilia Johansson)</td><td>female</td><td>38.0</td><td>1</td><td>5</td><td>347077</td><td>31.3875</td><td>null</td><td>S</td></tr><tr><td>29</td><td>1</td><td>3</td><td>\"O'Dwyer, Miss. Ellen \"\"Nellie\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330959</td><td>7.8792</td><td>null</td><td>Q</td></tr><tr><td>32</td><td>1</td><td>1</td><td>Spencer, Mrs. William Augustus (Marie Eugenie)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>PC 17569</td><td>146.5208</td><td>B78</td><td>C</td></tr><tr><td>33</td><td>1</td><td>3</td><td>Glynn, Miss. Mary Agatha</td><td>female</td><td>null</td><td>0</td><td>0</td><td>335677</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>39</td><td>0</td><td>3</td><td>Vander Planke, Miss. Augusta Maria</td><td>female</td><td>18.0</td><td>2</td><td>0</td><td>345764</td><td>18.0</td><td>null</td><td>S</td></tr><tr><td>40</td><td>1</td><td>3</td><td>Nicola-Yarred, Miss. Jamila</td><td>female</td><td>14.0</td><td>1</td><td>0</td><td>2651</td><td>11.2417</td><td>null</td><td>C</td></tr><tr><td>41</td><td>0</td><td>3</td><td>Ahlin, Mrs. Johan (Johanna Persdotter Larsson)</td><td>female</td><td>40.0</td><td>1</td><td>0</td><td>7546</td><td>9.475</td><td>null</td><td>S</td></tr><tr><td>42</td><td>0</td><td>2</td><td>Turpin, Mrs. William John Robert (Dorothy Ann Wonnacott)</td><td>female</td><td>27.0</td><td>1</td><td>0</td><td>11668</td><td>21.0</td><td>null</td><td>S</td></tr><tr><td>44</td><td>1</td><td>2</td><td>Laroche, Miss. Simonne Marie Anne Andree</td><td>female</td><td>3.0</td><td>1</td><td>2</td><td>SC/Paris 2123</td><td>41.5792</td><td>null</td><td>C</td></tr><tr><td>45</td><td>1</td><td>3</td><td>Devaney, Miss. Margaret Delia</td><td>female</td><td>19.0</td><td>0</td><td>0</td><td>330958</td><td>7.8792</td><td>null</td><td>Q</td></tr><tr><td>48</td><td>1</td><td>3</td><td>O'Driscoll, Miss. Bridget</td><td>female</td><td>null</td><td>0</td><td>0</td><td>14311</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>50</td><td>0</td><td>3</td><td>Arnold-Franchi, Mrs. Josef (Josefine Franchi)</td><td>female</td><td>18.0</td><td>1</td><td>0</td><td>349237</td><td>17.8</td><td>null</td><td>S</td></tr><tr><td>53</td><td>1</td><td>1</td><td>Harper, Mrs. Henry Sleeper (Myna Haxtun)</td><td>female</td><td>49.0</td><td>1</td><td>0</td><td>PC 17572</td><td>76.7292</td><td>D33</td><td>C</td></tr><tr><td>54</td><td>1</td><td>2</td><td>Faunthorpe, Mrs. Lizzie (Elizabeth Anne Wilkinson)</td><td>female</td><td>29.0</td><td>1</td><td>0</td><td>2926</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>57</td><td>1</td><td>2</td><td>Rugg, Miss. Emily</td><td>female</td><td>21.0</td><td>0</td><td>0</td><td>C.A. 31026</td><td>10.5</td><td>null</td><td>S</td></tr><tr><td>59</td><td>1</td><td>2</td><td>West, Miss. Constance Mirium</td><td>female</td><td>5.0</td><td>1</td><td>2</td><td>C.A. 34651</td><td>27.75</td><td>null</td><td>S</td></tr><tr><td>62</td><td>1</td><td>1</td><td>Icard, Miss. Amelie</td><td>female</td><td>38.0</td><td>0</td><td>0</td><td>113572</td><td>80.0</td><td>B28</td><td>null</td></tr><tr><td>67</td><td>1</td><td>2</td><td>Nye, Mrs. (Elizabeth Ramell)</td><td>female</td><td>29.0</td><td>0</td><td>0</td><td>C.A. 29395</td><td>10.5</td><td>F33</td><td>S</td></tr><tr><td>69</td><td>1</td><td>3</td><td>Andersson, Miss. Erna Alexandra</td><td>female</td><td>17.0</td><td>4</td><td>2</td><td>3101281</td><td>7.925</td><td>null</td><td>S</td></tr><tr><td>72</td><td>0</td><td>3</td><td>Goodwin, Miss. Lillian Amy</td><td>female</td><td>16.0</td><td>5</td><td>2</td><td>CA 2144</td><td>46.9</td><td>null</td><td>S</td></tr><tr><td>80</td><td>1</td><td>3</td><td>Dowdell, Miss. Elizabeth</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>364516</td><td>12.475</td><td>null</td><td>S</td></tr><tr><td>83</td><td>1</td><td>3</td><td>McDermott, Miss. Brigdet Delia</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330932</td><td>7.7875</td><td>null</td><td>Q</td></tr><tr><td>85</td><td>1</td><td>2</td><td>Ilett, Miss. Bertha</td><td>female</td><td>17.0</td><td>0</td><td>0</td><td>SO/C 14885</td><td>10.5</td><td>null</td><td>S</td></tr><tr><td>86</td><td>1</td><td>3</td><td>Backstrom, Mrs. Karl Alfred (Maria Mathilda Gustafsson)</td><td>female</td><td>33.0</td><td>3</td><td>0</td><td>3101278</td><td>15.85</td><td>null</td><td>S</td></tr><tr><td>89</td><td>1</td><td>1</td><td>Fortune, Miss. Mabel Helen</td><td>female</td><td>23.0</td><td>3</td><td>2</td><td>19950</td><td>263.0</td><td>C23 C25 C27</td><td>S</td></tr><tr><td>99</td><td>1</td><td>2</td><td>Doling, Mrs. John T (Ada Julia Bone)</td><td>female</td><td>34.0</td><td>0</td><td>1</td><td>231919</td><td>23.0</td><td>null</td><td>S</td></tr><tr><td>101</td><td>0</td><td>3</td><td>Petranec, Miss. Matilda</td><td>female</td><td>28.0</td><td>0</td><td>0</td><td>349245</td><td>7.8958</td><td>null</td><td>S</td></tr><tr><td>107</td><td>1</td><td>3</td><td>Salkjelsvik, Miss. Anna Kristine</td><td>female</td><td>21.0</td><td>0</td><td>0</td><td>343120</td><td>7.65</td><td>null</td><td>S</td></tr><tr><td>110</td><td>1</td><td>3</td><td>Moran, Miss. Bertha</td><td>female</td><td>null</td><td>1</td><td>0</td><td>371110</td><td>24.15</td><td>null</td><td>Q</td></tr><tr><td>112</td><td>0</td><td>3</td><td>Zabour, Miss. Hileni</td><td>female</td><td>14.5</td><td>1</td><td>0</td><td>2665</td><td>14.4542</td><td>null</td><td>C</td></tr><tr><td>114</td><td>0</td><td>3</td><td>Jussila, Miss. Katriina</td><td>female</td><td>20.0</td><td>1</td><td>0</td><td>4136</td><td>9.825</td><td>null</td><td>S</td></tr><tr><td>115</td><td>0</td><td>3</td><td>Attalah, Miss. Malake</td><td>female</td><td>17.0</td><td>0</td><td>0</td><td>2627</td><td>14.4583</td><td>null</td><td>C</td></tr><tr><td>120</td><td>0</td><td>3</td><td>Andersson, Miss. Ellis Anna Maria</td><td>female</td><td>2.0</td><td>4</td><td>2</td><td>347082</td><td>31.275</td><td>null</td><td>S</td></tr><tr><td>124</td><td>1</td><td>2</td><td>Webber, Miss. Susan</td><td>female</td><td>32.5</td><td>0</td><td>0</td><td>27267</td><td>13.0</td><td>E101</td><td>S</td></tr><tr><td>129</td><td>1</td><td>3</td><td>Peter, Miss. Anna</td><td>female</td><td>null</td><td>1</td><td>1</td><td>2668</td><td>22.3583</td><td>F E69</td><td>C</td></tr><tr><td>133</td><td>0</td><td>3</td><td>Robins, Mrs. Alexander A (Grace Charity Laury)</td><td>female</td><td>47.0</td><td>1</td><td>0</td><td>A/5. 3337</td><td>14.5</td><td>null</td><td>S</td></tr><tr><td>134</td><td>1</td><td>2</td><td>Weisz, Mrs. Leopold (Mathilde Francoise Pede)</td><td>female</td><td>29.0</td><td>1</td><td>0</td><td>228414</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>137</td><td>1</td><td>1</td><td>Newsom, Miss. Helen Monypeny</td><td>female</td><td>19.0</td><td>0</td><td>2</td><td>11752</td><td>26.2833</td><td>D47</td><td>S</td></tr><tr><td>141</td><td>0</td><td>3</td><td>Boulos, Mrs. Joseph (Sultana)</td><td>female</td><td>null</td><td>0</td><td>2</td><td>2678</td><td>15.2458</td><td>null</td><td>C</td></tr><tr><td>142</td><td>1</td><td>3</td><td>Nysten, Miss. Anna Sofia</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>347081</td><td>7.75</td><td>null</td><td>S</td></tr><tr><td>143</td><td>1</td><td>3</td><td>Hakkarainen, Mrs. Pekka Pietari (Elin Matilda Dolck)</td><td>female</td><td>24.0</td><td>1</td><td>0</td><td>STON/O2. 3101279</td><td>15.85</td><td>null</td><td>S</td></tr><tr><td>148</td><td>0</td><td>3</td><td>\"Ford, Miss. Robina Maggie \"\"Ruby\"\"\"</td><td>female</td><td>9.0</td><td>2</td><td>2</td><td>W./C. 6608</td><td>34.375</td><td>null</td><td>S</td></tr><tr><td>152</td><td>1</td><td>1</td><td>Pears, Mrs. Thomas (Edith Wearne)</td><td>female</td><td>22.0</td><td>1</td><td>0</td><td>113776</td><td>66.6</td><td>C2</td><td>S</td></tr><tr><td>157</td><td>1</td><td>3</td><td>\"Gilnagh, Miss. Katherine \"\"Katie\"\"\"</td><td>female</td><td>16.0</td><td>0</td><td>0</td><td>35851</td><td>7.7333</td><td>null</td><td>Q</td></tr><tr><td>162</td><td>1</td><td>2</td><td>\"Watt, Mrs. James (Elizabeth \"\"Bessie\"\" Inglis Milne)\"</td><td>female</td><td>40.0</td><td>0</td><td>0</td><td>C.A. 33595</td><td>15.75</td><td>null</td><td>S</td></tr><tr><td>167</td><td>1</td><td>1</td><td>Chibnall, Mrs. (Edith Martha Bowerman)</td><td>female</td><td>null</td><td>0</td><td>1</td><td>113505</td><td>55.0</td><td>E33</td><td>S</td></tr><tr><td>168</td><td>0</td><td>3</td><td>Skoog, Mrs. William (Anna Bernhardina Karlsson)</td><td>female</td><td>45.0</td><td>1</td><td>4</td><td>347088</td><td>27.9</td><td>null</td><td>S</td></tr><tr><td>173</td><td>1</td><td>3</td><td>Johnson, Miss. Eleanor Ileen</td><td>female</td><td>1.0</td><td>1</td><td>1</td><td>347742</td><td>11.1333</td><td>null</td><td>S</td></tr><tr><td>178</td><td>0</td><td>1</td><td>Isham, Miss. Ann Elizabeth</td><td>female</td><td>50.0</td><td>0</td><td>0</td><td>PC 17595</td><td>28.7125</td><td>C49</td><td>C</td></tr><tr><td>181</td><td>0</td><td>3</td><td>Sage, Miss. Constance Gladys</td><td>female</td><td>null</td><td>8</td><td>2</td><td>CA. 2343</td><td>69.55</td><td>null</td><td>S</td></tr><tr><td>185</td><td>1</td><td>3</td><td>Kink-Heilmann, Miss. Luise Gretchen</td><td>female</td><td>4.0</td><td>0</td><td>2</td><td>315153</td><td>22.025</td><td>null</td><td>S</td></tr><tr><td>187</td><td>1</td><td>3</td><td>\"O'Brien, Mrs. Thomas (Johanna \"\"Hannah\"\" Godfrey)\"</td><td>female</td><td>null</td><td>1</td><td>0</td><td>370365</td><td>15.5</td><td>null</td><td>Q</td></tr><tr><td>191</td><td>1</td><td>2</td><td>Pinsky, Mrs. (Rosa)</td><td>female</td><td>32.0</td><td>0</td><td>0</td><td>234604</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>193</td><td>1</td><td>3</td><td>Andersen-Jensen, Miss. Carla Christine Nielsine</td><td>female</td><td>19.0</td><td>1</td><td>0</td><td>350046</td><td>7.8542</td><td>null</td><td>S</td></tr><tr><td>195</td><td>1</td><td>1</td><td>Brown, Mrs. James Joseph (Margaret Tobin)</td><td>female</td><td>44.0</td><td>0</td><td>0</td><td>PC 17610</td><td>27.7208</td><td>B4</td><td>C</td></tr><tr><td>196</td><td>1</td><td>1</td><td>Lurette, Miss. Elise</td><td>female</td><td>58.0</td><td>0</td><td>0</td><td>PC 17569</td><td>146.5208</td><td>B80</td><td>C</td></tr><tr><td>199</td><td>1</td><td>3</td><td>\"Madigan, Miss. Margaret \"\"Maggie\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>370370</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>200</td><td>0</td><td>2</td><td>\"Yrois, Miss. Henriette (\"\"Mrs Harbeck\"\")\"</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>248747</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>206</td><td>0</td><td>3</td><td>Strom, Miss. Telma Matilda</td><td>female</td><td>2.0</td><td>0</td><td>1</td><td>347054</td><td>10.4625</td><td>G6</td><td>S</td></tr><tr><td>209</td><td>1</td><td>3</td><td>\"Carr, Miss. Helen \"\"Ellen\"\"\"</td><td>female</td><td>16.0</td><td>0</td><td>0</td><td>367231</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>212</td><td>1</td><td>2</td><td>Cameron, Miss. Clear Annie</td><td>female</td><td>35.0</td><td>0</td><td>0</td><td>F.C.C. 13528</td><td>21.0</td><td>null</td><td>S</td></tr><tr><td>216</td><td>1</td><td>1</td><td>Newell, Miss. Madeleine</td><td>female</td><td>31.0</td><td>1</td><td>0</td><td>35273</td><td>113.275</td><td>D36</td><td>C</td></tr><tr><td>217</td><td>1</td><td>3</td><td>Honkanen, Miss. Eliina</td><td>female</td><td>27.0</td><td>0</td><td>0</td><td>STON/O2. 3101283</td><td>7.925</td><td>null</td><td>S</td></tr><tr><td>219</td><td>1</td><td>1</td><td>Bazzani, Miss. Albina</td><td>female</td><td>32.0</td><td>0</td><td>0</td><td>11813</td><td>76.2917</td><td>D15</td><td>C</td></tr><tr><td>230</td><td>0</td><td>3</td><td>Lefebre, Miss. Mathilde</td><td>female</td><td>null</td><td>3</td><td>1</td><td>4133</td><td>25.4667</td><td>null</td><td>S</td></tr><tr><td>231</td><td>1</td><td>1</td><td>Harris, Mrs. Henry Birkhardt (Irene Wallach)</td><td>female</td><td>35.0</td><td>1</td><td>0</td><td>36973</td><td>83.475</td><td>C83</td><td>S</td></tr><tr><td>234</td><td>1</td><td>3</td><td>Asplund, Miss. Lillian Gertrud</td><td>female</td><td>5.0</td><td>4</td><td>2</td><td>347077</td><td>31.3875</td><td>null</td><td>S</td></tr><tr><td>236</td><td>0</td><td>3</td><td>Harknett, Miss. Alice Phoebe</td><td>female</td><td>null</td><td>0</td><td>0</td><td>W./C. 6609</td><td>7.55</td><td>null</td><td>S</td></tr><tr><td>238</td><td>1</td><td>2</td><td>\"Collyer, Miss. Marjorie \"\"Lottie\"\"\"</td><td>female</td><td>8.0</td><td>0</td><td>2</td><td>C.A. 31921</td><td>26.25</td><td>null</td><td>S</td></tr><tr><td>241</td><td>0</td><td>3</td><td>Zabour, Miss. Thamine</td><td>female</td><td>null</td><td>1</td><td>0</td><td>2665</td><td>14.4542</td><td>null</td><td>C</td></tr><tr><td>242</td><td>1</td><td>3</td><td>\"Murphy, Miss. Katherine \"\"Kate\"\"\"</td><td>female</td><td>null</td><td>1</td><td>0</td><td>367230</td><td>15.5</td><td>null</td><td>Q</td></tr><tr><td>247</td><td>0</td><td>3</td><td>Lindahl, Miss. Agda Thorilda Viktoria</td><td>female</td><td>25.0</td><td>0</td><td>0</td><td>347071</td><td>7.775</td><td>null</td><td>S</td></tr><tr><td>248</td><td>1</td><td>2</td><td>Hamalainen, Mrs. William (Anna)</td><td>female</td><td>24.0</td><td>0</td><td>2</td><td>250649</td><td>14.5</td><td>null</td><td>S</td></tr><tr><td>252</td><td>0</td><td>3</td><td>Strom, Mrs. Wilhelm (Elna Matilda Persson)</td><td>female</td><td>29.0</td><td>1</td><td>1</td><td>347054</td><td>10.4625</td><td>G6</td><td>S</td></tr><tr><td>255</td><td>0</td><td>3</td><td>Rosblom, Mrs. Viktor (Helena Wilhelmina)</td><td>female</td><td>41.0</td><td>0</td><td>2</td><td>370129</td><td>20.2125</td><td>null</td><td>S</td></tr><tr><td>256</td><td>1</td><td>3</td><td>Touma, Mrs. Darwis (Hanne Youssef Razi)</td><td>female</td><td>29.0</td><td>0</td><td>2</td><td>2650</td><td>15.2458</td><td>null</td><td>C</td></tr><tr><td>257</td><td>1</td><td>1</td><td>Thorne, Mrs. Gertrude Maybelle</td><td>female</td><td>null</td><td>0</td><td>0</td><td>PC 17585</td><td>79.2</td><td>null</td><td>C</td></tr><tr><td>258</td><td>1</td><td>1</td><td>Cherry, Miss. Gladys</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>110152</td><td>86.5</td><td>B77</td><td>S</td></tr><tr><td>259</td><td>1</td><td>1</td><td>Ward, Miss. Anna</td><td>female</td><td>35.0</td><td>0</td><td>0</td><td>PC 17755</td><td>512.3292</td><td>null</td><td>C</td></tr><tr><td>260</td><td>1</td><td>2</td><td>Parrish, Mrs. (Lutie Davis)</td><td>female</td><td>50.0</td><td>0</td><td>1</td><td>230433</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>265</td><td>0</td><td>3</td><td>Henry, Miss. Delia</td><td>female</td><td>null</td><td>0</td><td>0</td><td>382649</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>269</td><td>1</td><td>1</td><td>Graham, Mrs. William Thompson (Edith Junkins)</td><td>female</td><td>58.0</td><td>0</td><td>1</td><td>PC 17582</td><td>153.4625</td><td>C125</td><td>S</td></tr><tr><td>270</td><td>1</td><td>1</td><td>Bissette, Miss. Amelia</td><td>female</td><td>35.0</td><td>0</td><td>0</td><td>PC 17760</td><td>135.6333</td><td>C99</td><td>S</td></tr><tr><td>273</td><td>1</td><td>2</td><td>Mellinger, Mrs. (Elizabeth Anne Maidment)</td><td>female</td><td>41.0</td><td>0</td><td>1</td><td>250644</td><td>19.5</td><td>null</td><td>S</td></tr><tr><td>275</td><td>1</td><td>3</td><td>\"Healy, Miss. Hanora \"\"Nora\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>370375</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>276</td><td>1</td><td>1</td><td>Andrews, Miss. Kornelia Theodosia</td><td>female</td><td>63.0</td><td>1</td><td>0</td><td>13502</td><td>77.9583</td><td>D7</td><td>S</td></tr><tr><td>277</td><td>0</td><td>3</td><td>Lindblom, Miss. Augusta Charlotta</td><td>female</td><td>45.0</td><td>0</td><td>0</td><td>347073</td><td>7.75</td><td>null</td><td>S</td></tr><tr><td>280</td><td>1</td><td>3</td><td>Abbott, Mrs. Stanton (Rosa Hunt)</td><td>female</td><td>35.0</td><td>1</td><td>1</td><td>C.A. 2673</td><td>20.25</td><td>null</td><td>S</td></tr><tr><td>290</td><td>1</td><td>3</td><td>Connolly, Miss. Kate</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>370373</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>291</td><td>1</td><td>1</td><td>\"Barber, Miss. Ellen \"\"Nellie\"\"\"</td><td>female</td><td>26.0</td><td>0</td><td>0</td><td>19877</td><td>78.85</td><td>null</td><td>S</td></tr><tr><td>292</td><td>1</td><td>1</td><td>Bishop, Mrs. Dickinson H (Helen Walton)</td><td>female</td><td>19.0</td><td>1</td><td>0</td><td>11967</td><td>91.0792</td><td>B49</td><td>C</td></tr><tr><td>294</td><td>0</td><td>3</td><td>Haas, Miss. Aloisia</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>349236</td><td>8.85</td><td>null</td><td>S</td></tr><tr><td>298</td><td>0</td><td>1</td><td>Allison, Miss. Helen Loraine</td><td>female</td><td>2.0</td><td>1</td><td>2</td><td>113781</td><td>151.55</td><td>C22 C26</td><td>S</td></tr><tr><td>300</td><td>1</td><td>1</td><td>Baxter, Mrs. James (Helene DeLaudeniere Chaput)</td><td>female</td><td>50.0</td><td>0</td><td>1</td><td>PC 17558</td><td>247.5208</td><td>B58 B60</td><td>C</td></tr><tr><td>301</td><td>1</td><td>3</td><td>\"Kelly, Miss. Anna Katherine \"\"Annie Kate\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>9234</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>304</td><td>1</td><td>2</td><td>Keane, Miss. Nora A</td><td>female</td><td>null</td><td>0</td><td>0</td><td>226593</td><td>12.35</td><td>E101</td><td>Q</td></tr><tr><td>307</td><td>1</td><td>1</td><td>Fleming, Miss. Margaret</td><td>female</td><td>null</td><td>0</td><td>0</td><td>17421</td><td>110.8833</td><td>null</td><td>C</td></tr><tr><td>308</td><td>1</td><td>1</td><td>Penasco y Castellana, Mrs. Victor de Satode (Maria Josefa Perez de Soto y Vallejo)</td><td>female</td><td>17.0</td><td>1</td><td>0</td><td>PC 17758</td><td>108.9</td><td>C65</td><td>C</td></tr><tr><td>310</td><td>1</td><td>1</td><td>Francatelli, Miss. Laura Mabel</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>PC 17485</td><td>56.9292</td><td>E36</td><td>C</td></tr><tr><td>311</td><td>1</td><td>1</td><td>Hays, Miss. Margaret Bechstein</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>11767</td><td>83.1583</td><td>C54</td><td>C</td></tr><tr><td>312</td><td>1</td><td>1</td><td>Ryerson, Miss. Emily Borie</td><td>female</td><td>18.0</td><td>2</td><td>2</td><td>PC 17608</td><td>262.375</td><td>B57 B59 B63 B66</td><td>C</td></tr><tr><td>313</td><td>0</td><td>2</td><td>Lahtinen, Mrs. William (Anna Sylfven)</td><td>female</td><td>26.0</td><td>1</td><td>1</td><td>250651</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>316</td><td>1</td><td>3</td><td>Nilsson, Miss. Helmina Josefina</td><td>female</td><td>26.0</td><td>0</td><td>0</td><td>347470</td><td>7.8542</td><td>null</td><td>S</td></tr><tr><td>317</td><td>1</td><td>2</td><td>Kantor, Mrs. Sinai (Miriam Sternin)</td><td>female</td><td>24.0</td><td>1</td><td>0</td><td>244367</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>319</td><td>1</td><td>1</td><td>Wick, Miss. Mary Natalie</td><td>female</td><td>31.0</td><td>0</td><td>2</td><td>36928</td><td>164.8667</td><td>C7</td><td>S</td></tr><tr><td>320</td><td>1</td><td>1</td><td>Spedden, Mrs. Frederic Oakley (Margaretta Corning Stone)</td><td>female</td><td>40.0</td><td>1</td><td>1</td><td>16966</td><td>134.5</td><td>E34</td><td>C</td></tr><tr><td>323</td><td>1</td><td>2</td><td>Slayter, Miss. Hilda Mary</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>234818</td><td>12.35</td><td>null</td><td>Q</td></tr><tr><td>324</td><td>1</td><td>2</td><td>Caldwell, Mrs. Albert Francis (Sylvia Mae Harbaugh)</td><td>female</td><td>22.0</td><td>1</td><td>1</td><td>248738</td><td>29.0</td><td>null</td><td>S</td></tr><tr><td>326</td><td>1</td><td>1</td><td>Young, Miss. Marie Grice</td><td>female</td><td>36.0</td><td>0</td><td>0</td><td>PC 17760</td><td>135.6333</td><td>C32</td><td>C</td></tr><tr><td>328</td><td>1</td><td>2</td><td>Ball, Mrs. (Ada E Hall)</td><td>female</td><td>36.0</td><td>0</td><td>0</td><td>28551</td><td>13.0</td><td>D</td><td>S</td></tr><tr><td>329</td><td>1</td><td>3</td><td>Goldsmith, Mrs. Frank John (Emily Alice Brown)</td><td>female</td><td>31.0</td><td>1</td><td>1</td><td>363291</td><td>20.525</td><td>null</td><td>S</td></tr><tr><td>330</td><td>1</td><td>1</td><td>Hippach, Miss. Jean Gertrude</td><td>female</td><td>16.0</td><td>0</td><td>1</td><td>111361</td><td>57.9792</td><td>B18</td><td>C</td></tr><tr><td>331</td><td>1</td><td>3</td><td>McCoy, Miss. Agnes</td><td>female</td><td>null</td><td>2</td><td>0</td><td>367226</td><td>23.25</td><td>null</td><td>Q</td></tr><tr><td>335</td><td>1</td><td>1</td><td>Frauenthal, Mrs. Henry William (Clara Heinsheimer)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>PC 17611</td><td>133.65</td><td>null</td><td>S</td></tr><tr><td>338</td><td>1</td><td>1</td><td>Burns, Miss. Elizabeth Margaret</td><td>female</td><td>41.0</td><td>0</td><td>0</td><td>16966</td><td>134.5</td><td>E40</td><td>C</td></tr><tr><td>342</td><td>1</td><td>1</td><td>Fortune, Miss. Alice Elizabeth</td><td>female</td><td>24.0</td><td>3</td><td>2</td><td>19950</td><td>263.0</td><td>C23 C25 C27</td><td>S</td></tr><tr><td>346</td><td>1</td><td>2</td><td>\"Brown, Miss. Amelia \"\"Mildred\"\"\"</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>248733</td><td>13.0</td><td>F33</td><td>S</td></tr><tr><td>347</td><td>1</td><td>2</td><td>Smith, Miss. Marion Elsie</td><td>female</td><td>40.0</td><td>0</td><td>0</td><td>31418</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>348</td><td>1</td><td>3</td><td>Davison, Mrs. Thomas Henry (Mary E Finck)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>386525</td><td>16.1</td><td>null</td><td>S</td></tr><tr><td>357</td><td>1</td><td>1</td><td>Bowerman, Miss. Elsie Edith</td><td>female</td><td>22.0</td><td>0</td><td>1</td><td>113505</td><td>55.0</td><td>E33</td><td>S</td></tr><tr><td>358</td><td>0</td><td>2</td><td>Funk, Miss. Annie Clemmer</td><td>female</td><td>38.0</td><td>0</td><td>0</td><td>237671</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>359</td><td>1</td><td>3</td><td>McGovern, Miss. Mary</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330931</td><td>7.8792</td><td>null</td><td>Q</td></tr><tr><td>360</td><td>1</td><td>3</td><td>\"Mockler, Miss. Helen Mary \"\"Ellie\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330980</td><td>7.8792</td><td>null</td><td>Q</td></tr><tr><td>363</td><td>0</td><td>3</td><td>Barbara, Mrs. (Catherine David)</td><td>female</td><td>45.0</td><td>0</td><td>1</td><td>2691</td><td>14.4542</td><td>null</td><td>C</td></tr><tr><td>367</td><td>1</td><td>1</td><td>Warren, Mrs. Frank Manley (Anna Sophia Atkinson)</td><td>female</td><td>60.0</td><td>1</td><td>0</td><td>110813</td><td>75.25</td><td>D37</td><td>C</td></tr><tr><td>368</td><td>1</td><td>3</td><td>Moussa, Mrs. (Mantoura Boulos)</td><td>female</td><td>null</td><td>0</td><td>0</td><td>2626</td><td>7.2292</td><td>null</td><td>C</td></tr><tr><td>369</td><td>1</td><td>3</td><td>Jermyn, Miss. Annie</td><td>female</td><td>null</td><td>0</td><td>0</td><td>14313</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>370</td><td>1</td><td>1</td><td>Aubart, Mme. Leontine Pauline</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>PC 17477</td><td>69.3</td><td>B35</td><td>C</td></tr><tr><td>375</td><td>0</td><td>3</td><td>Palsson, Miss. Stina Viola</td><td>female</td><td>3.0</td><td>3</td><td>1</td><td>349909</td><td>21.075</td><td>null</td><td>S</td></tr><tr><td>376</td><td>1</td><td>1</td><td>Meyer, Mrs. Edgar Joseph (Leila Saks)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>PC 17604</td><td>82.1708</td><td>null</td><td>C</td></tr><tr><td>377</td><td>1</td><td>3</td><td>Landergren, Miss. Aurora Adelia</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>C 7077</td><td>7.25</td><td>null</td><td>S</td></tr><tr><td>381</td><td>1</td><td>1</td><td>Bidois, Miss. Rosalie</td><td>female</td><td>42.0</td><td>0</td><td>0</td><td>PC 17757</td><td>227.525</td><td>null</td><td>C</td></tr><tr><td>382</td><td>1</td><td>3</td><td>\"Nakid, Miss. Maria (\"\"Mary\"\")\"</td><td>female</td><td>1.0</td><td>0</td><td>2</td><td>2653</td><td>15.7417</td><td>null</td><td>C</td></tr><tr><td>384</td><td>1</td><td>1</td><td>Holverson, Mrs. Alexander Oskar (Mary Aline Towner)</td><td>female</td><td>35.0</td><td>1</td><td>0</td><td>113789</td><td>52.0</td><td>null</td><td>S</td></tr><tr><td>388</td><td>1</td><td>2</td><td>Buss, Miss. Kate</td><td>female</td><td>36.0</td><td>0</td><td>0</td><td>27849</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>390</td><td>1</td><td>2</td><td>Lehmann, Miss. Bertha</td><td>female</td><td>17.0</td><td>0</td><td>0</td><td>SC 1748</td><td>12.0</td><td>null</td><td>C</td></tr><tr><td>394</td><td>1</td><td>1</td><td>Newell, Miss. Marjorie</td><td>female</td><td>23.0</td><td>1</td><td>0</td><td>35273</td><td>113.275</td><td>D36</td><td>C</td></tr><tr><td>395</td><td>1</td><td>3</td><td>Sandstrom, Mrs. Hjalmar (Agnes Charlotta Bengtsson)</td><td>female</td><td>24.0</td><td>0</td><td>2</td><td>PP 9549</td><td>16.7</td><td>G6</td><td>S</td></tr><tr><td>397</td><td>0</td><td>3</td><td>Olsson, Miss. Elina</td><td>female</td><td>31.0</td><td>0</td><td>0</td><td>350407</td><td>7.8542</td><td>null</td><td>S</td></tr><tr><td>400</td><td>1</td><td>2</td><td>Trout, Mrs. William H (Jessie L)</td><td>female</td><td>28.0</td><td>0</td><td>0</td><td>240929</td><td>12.65</td><td>null</td><td>S</td></tr><tr><td>403</td><td>0</td><td>3</td><td>Jussila, Miss. Mari Aina</td><td>female</td><td>21.0</td><td>1</td><td>0</td><td>4137</td><td>9.825</td><td>null</td><td>S</td></tr><tr><td>405</td><td>0</td><td>3</td><td>Oreskovic, Miss. Marija</td><td>female</td><td>20.0</td><td>0</td><td>0</td><td>315096</td><td>8.6625</td><td>null</td><td>S</td></tr><tr><td>410</td><td>0</td><td>3</td><td>Lefebre, Miss. Ida</td><td>female</td><td>null</td><td>3</td><td>1</td><td>4133</td><td>25.4667</td><td>null</td><td>S</td></tr><tr><td>413</td><td>1</td><td>1</td><td>Minahan, Miss. Daisy E</td><td>female</td><td>33.0</td><td>1</td><td>0</td><td>19928</td><td>90.0</td><td>C78</td><td>Q</td></tr><tr><td>416</td><td>0</td><td>3</td><td>Meek, Mrs. Thomas (Annie Louise Rowley)</td><td>female</td><td>null</td><td>0</td><td>0</td><td>343095</td><td>8.05</td><td>null</td><td>S</td></tr><tr><td>417</td><td>1</td><td>2</td><td>Drew, Mrs. James Vivian (Lulu Thorne Christian)</td><td>female</td><td>34.0</td><td>1</td><td>1</td><td>28220</td><td>32.5</td><td>null</td><td>S</td></tr><tr><td>418</td><td>1</td><td>2</td><td>Silven, Miss. Lyyli Karoliina</td><td>female</td><td>18.0</td><td>0</td><td>2</td><td>250652</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>420</td><td>0</td><td>3</td><td>Van Impe, Miss. Catharina</td><td>female</td><td>10.0</td><td>0</td><td>2</td><td>345773</td><td>24.15</td><td>null</td><td>S</td></tr><tr><td>424</td><td>0</td><td>3</td><td>Danbom, Mrs. Ernst Gilbert (Anna Sigrid Maria Brogren)</td><td>female</td><td>28.0</td><td>1</td><td>1</td><td>347080</td><td>14.4</td><td>null</td><td>S</td></tr><tr><td>427</td><td>1</td><td>2</td><td>Clarke, Mrs. Charles V (Ada Maria Winfield)</td><td>female</td><td>28.0</td><td>1</td><td>0</td><td>2003</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>428</td><td>1</td><td>2</td><td>\"Phillips, Miss. Kate Florence (\"\"Mrs Kate Louise Phillips Marshall\"\")\"</td><td>female</td><td>19.0</td><td>0</td><td>0</td><td>250655</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>432</td><td>1</td><td>3</td><td>Thorneycroft, Mrs. Percival (Florence Kate White)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>376564</td><td>16.1</td><td>null</td><td>S</td></tr><tr><td>433</td><td>1</td><td>2</td><td>Louch, Mrs. Charles Alexander (Alice Adelaide Slow)</td><td>female</td><td>42.0</td><td>1</td><td>0</td><td>SC/AH 3085</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>436</td><td>1</td><td>1</td><td>Carter, Miss. Lucile Polk</td><td>female</td><td>14.0</td><td>1</td><td>2</td><td>113760</td><td>120.0</td><td>B96 B98</td><td>S</td></tr><tr><td>437</td><td>0</td><td>3</td><td>\"Ford, Miss. Doolina Margaret \"\"Daisy\"\"\"</td><td>female</td><td>21.0</td><td>2</td><td>2</td><td>W./C. 6608</td><td>34.375</td><td>null</td><td>S</td></tr><tr><td>438</td><td>1</td><td>2</td><td>Richards, Mrs. Sidney (Emily Hocking)</td><td>female</td><td>24.0</td><td>2</td><td>3</td><td>29106</td><td>18.75</td><td>null</td><td>S</td></tr><tr><td>441</td><td>1</td><td>2</td><td>Hart, Mrs. Benjamin (Esther Ada Bloomfield)</td><td>female</td><td>45.0</td><td>1</td><td>1</td><td>F.C.C. 13529</td><td>26.25</td><td>null</td><td>S</td></tr><tr><td>444</td><td>1</td><td>2</td><td>Reynaldo, Ms. Encarnacion</td><td>female</td><td>28.0</td><td>0</td><td>0</td><td>230434</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>447</td><td>1</td><td>2</td><td>Mellinger, Miss. Madeleine Violet</td><td>female</td><td>13.0</td><td>0</td><td>1</td><td>250644</td><td>19.5</td><td>null</td><td>S</td></tr><tr><td>449</td><td>1</td><td>3</td><td>Baclini, Miss. Marie Catherine</td><td>female</td><td>5.0</td><td>2</td><td>1</td><td>2666</td><td>19.2583</td><td>null</td><td>C</td></tr><tr><td>458</td><td>1</td><td>1</td><td>Kenyon, Mrs. Frederick R (Marion)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>17464</td><td>51.8625</td><td>D21</td><td>S</td></tr><tr><td>459</td><td>1</td><td>2</td><td>Toomey, Miss. Ellen</td><td>female</td><td>50.0</td><td>0</td><td>0</td><td>F.C.C. 13531</td><td>10.5</td><td>null</td><td>S</td></tr><tr><td>470</td><td>1</td><td>3</td><td>Baclini, Miss. Helene Barbara</td><td>female</td><td>0.75</td><td>2</td><td>1</td><td>2666</td><td>19.2583</td><td>null</td><td>C</td></tr><tr><td>473</td><td>1</td><td>2</td><td>West, Mrs. Edwy Arthur (Ada Mary Worth)</td><td>female</td><td>33.0</td><td>1</td><td>2</td><td>C.A. 34651</td><td>27.75</td><td>null</td><td>S</td></tr><tr><td>474</td><td>1</td><td>2</td><td>Jerwan, Mrs. Amin S (Marie Marthe Thuillard)</td><td>female</td><td>23.0</td><td>0</td><td>0</td><td>SC/AH Basle 541</td><td>13.7917</td><td>D</td><td>C</td></tr><tr><td>475</td><td>0</td><td>3</td><td>Strandberg, Miss. Ida Sofia</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>7553</td><td>9.8375</td><td>null</td><td>S</td></tr><tr><td>480</td><td>1</td><td>3</td><td>Hirvonen, Miss. Hildur E</td><td>female</td><td>2.0</td><td>0</td><td>1</td><td>3101298</td><td>12.2875</td><td>null</td><td>S</td></tr><tr><td>484</td><td>1</td><td>3</td><td>Turkula, Mrs. (Hedwig)</td><td>female</td><td>63.0</td><td>0</td><td>0</td><td>4134</td><td>9.5875</td><td>null</td><td>S</td></tr><tr><td>486</td><td>0</td><td>3</td><td>Lefebre, Miss. Jeannie</td><td>female</td><td>null</td><td>3</td><td>1</td><td>4133</td><td>25.4667</td><td>null</td><td>S</td></tr><tr><td>487</td><td>1</td><td>1</td><td>Hoyt, Mrs. Frederick Maxfield (Jane Anne Forby)</td><td>female</td><td>35.0</td><td>1</td><td>0</td><td>19943</td><td>90.0</td><td>C93</td><td>S</td></tr><tr><td>497</td><td>1</td><td>1</td><td>Eustis, Miss. Elizabeth Mussey</td><td>female</td><td>54.0</td><td>1</td><td>0</td><td>36947</td><td>78.2667</td><td>D20</td><td>C</td></tr><tr><td>499</td><td>0</td><td>1</td><td>Allison, Mrs. Hudson J C (Bessie Waldo Daniels)</td><td>female</td><td>25.0</td><td>1</td><td>2</td><td>113781</td><td>151.55</td><td>C22 C26</td><td>S</td></tr><tr><td>502</td><td>0</td><td>3</td><td>Canavan, Miss. Mary</td><td>female</td><td>21.0</td><td>0</td><td>0</td><td>364846</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>503</td><td>0</td><td>3</td><td>O'Sullivan, Miss. Bridget Mary</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330909</td><td>7.6292</td><td>null</td><td>Q</td></tr><tr><td>504</td><td>0</td><td>3</td><td>Laitinen, Miss. Kristina Sofia</td><td>female</td><td>37.0</td><td>0</td><td>0</td><td>4135</td><td>9.5875</td><td>null</td><td>S</td></tr><tr><td>505</td><td>1</td><td>1</td><td>Maioni, Miss. Roberta</td><td>female</td><td>16.0</td><td>0</td><td>0</td><td>110152</td><td>86.5</td><td>B79</td><td>S</td></tr><tr><td>507</td><td>1</td><td>2</td><td>Quick, Mrs. Frederick Charles (Jane Richards)</td><td>female</td><td>33.0</td><td>0</td><td>2</td><td>26360</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>514</td><td>1</td><td>1</td><td>Rothschild, Mrs. Martin (Elizabeth L. Barrett)</td><td>female</td><td>54.0</td><td>1</td><td>0</td><td>PC 17603</td><td>59.4</td><td>null</td><td>C</td></tr><tr><td>517</td><td>1</td><td>2</td><td>Lemore, Mrs. (Amelia Milley)</td><td>female</td><td>34.0</td><td>0</td><td>0</td><td>C.A. 34260</td><td>10.5</td><td>F33</td><td>S</td></tr><tr><td>519</td><td>1</td><td>2</td><td>\"Angle, Mrs. William A (Florence \"\"Mary\"\" Agnes Hughes)\"</td><td>female</td><td>36.0</td><td>1</td><td>0</td><td>226875</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>521</td><td>1</td><td>1</td><td>Perreault, Miss. Anne</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>12749</td><td>93.5</td><td>B73</td><td>S</td></tr><tr><td>524</td><td>1</td><td>1</td><td>Hippach, Mrs. Louis Albert (Ida Sophia Fischer)</td><td>female</td><td>44.0</td><td>0</td><td>1</td><td>111361</td><td>57.9792</td><td>B18</td><td>C</td></tr><tr><td>527</td><td>1</td><td>2</td><td>Ridsdale, Miss. Lucy</td><td>female</td><td>50.0</td><td>0</td><td>0</td><td>W./C. 14258</td><td>10.5</td><td>null</td><td>S</td></tr><tr><td>531</td><td>1</td><td>2</td><td>Quick, Miss. Phyllis May</td><td>female</td><td>2.0</td><td>1</td><td>1</td><td>26360</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>534</td><td>1</td><td>3</td><td>Peter, Mrs. Catherine (Catherine Rizk)</td><td>female</td><td>null</td><td>0</td><td>2</td><td>2668</td><td>22.3583</td><td>null</td><td>C</td></tr><tr><td>535</td><td>0</td><td>3</td><td>Cacic, Miss. Marija</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>315084</td><td>8.6625</td><td>null</td><td>S</td></tr><tr><td>536</td><td>1</td><td>2</td><td>Hart, Miss. Eva Miriam</td><td>female</td><td>7.0</td><td>0</td><td>2</td><td>F.C.C. 13529</td><td>26.25</td><td>null</td><td>S</td></tr><tr><td>538</td><td>1</td><td>1</td><td>LeRoy, Miss. Bertha</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>PC 17761</td><td>106.425</td><td>null</td><td>C</td></tr><tr><td>540</td><td>1</td><td>1</td><td>Frolicher, Miss. Hedwig Margaritha</td><td>female</td><td>22.0</td><td>0</td><td>2</td><td>13568</td><td>49.5</td><td>B39</td><td>C</td></tr><tr><td>541</td><td>1</td><td>1</td><td>Crosby, Miss. Harriet R</td><td>female</td><td>36.0</td><td>0</td><td>2</td><td>WE/P 5735</td><td>71.0</td><td>B22</td><td>S</td></tr><tr><td>542</td><td>0</td><td>3</td><td>Andersson, Miss. Ingeborg Constanzia</td><td>female</td><td>9.0</td><td>4</td><td>2</td><td>347082</td><td>31.275</td><td>null</td><td>S</td></tr><tr><td>543</td><td>0</td><td>3</td><td>Andersson, Miss. Sigrid Elisabeth</td><td>female</td><td>11.0</td><td>4</td><td>2</td><td>347082</td><td>31.275</td><td>null</td><td>S</td></tr><tr><td>547</td><td>1</td><td>2</td><td>Beane, Mrs. Edward (Ethel Clarke)</td><td>female</td><td>19.0</td><td>1</td><td>0</td><td>2908</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>555</td><td>1</td><td>3</td><td>Ohman, Miss. Velin</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>347085</td><td>7.775</td><td>null</td><td>S</td></tr><tr><td>557</td><td>1</td><td>1</td><td>\"Duff Gordon, Lady. (Lucille Christiana Sutherland) (\"\"Mrs Morgan\"\")\"</td><td>female</td><td>48.0</td><td>1</td><td>0</td><td>11755</td><td>39.6</td><td>A16</td><td>C</td></tr><tr><td>559</td><td>1</td><td>1</td><td>Taussig, Mrs. Emil (Tillie Mandelbaum)</td><td>female</td><td>39.0</td><td>1</td><td>1</td><td>110413</td><td>79.65</td><td>E67</td><td>S</td></tr><tr><td>560</td><td>1</td><td>3</td><td>de Messemaeker, Mrs. Guillaume Joseph (Emma)</td><td>female</td><td>36.0</td><td>1</td><td>0</td><td>345572</td><td>17.4</td><td>null</td><td>S</td></tr><tr><td>565</td><td>0</td><td>3</td><td>Meanwell, Miss. (Marion Ogden)</td><td>female</td><td>null</td><td>0</td><td>0</td><td>SOTON/O.Q. 392087</td><td>8.05</td><td>null</td><td>S</td></tr><tr><td>568</td><td>0</td><td>3</td><td>Palsson, Mrs. Nils (Alma Cornelia Berglund)</td><td>female</td><td>29.0</td><td>0</td><td>4</td><td>349909</td><td>21.075</td><td>null</td><td>S</td></tr><tr><td>572</td><td>1</td><td>1</td><td>Appleton, Mrs. Edward Dale (Charlotte Lamson)</td><td>female</td><td>53.0</td><td>2</td><td>0</td><td>11769</td><td>51.4792</td><td>C101</td><td>S</td></tr><tr><td>574</td><td>1</td><td>3</td><td>Kelly, Miss. Mary</td><td>female</td><td>null</td><td>0</td><td>0</td><td>14312</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>577</td><td>1</td><td>2</td><td>Garside, Miss. Ethel</td><td>female</td><td>34.0</td><td>0</td><td>0</td><td>243880</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>578</td><td>1</td><td>1</td><td>Silvey, Mrs. William Baird (Alice Munger)</td><td>female</td><td>39.0</td><td>1</td><td>0</td><td>13507</td><td>55.9</td><td>E44</td><td>S</td></tr><tr><td>579</td><td>0</td><td>3</td><td>Caram, Mrs. Joseph (Maria Elias)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>2689</td><td>14.4583</td><td>null</td><td>C</td></tr><tr><td>581</td><td>1</td><td>2</td><td>Christy, Miss. Julie Rachel</td><td>female</td><td>25.0</td><td>1</td><td>1</td><td>237789</td><td>30.0</td><td>null</td><td>S</td></tr><tr><td>582</td><td>1</td><td>1</td><td>Thayer, Mrs. John Borland (Marian Longstreth Morris)</td><td>female</td><td>39.0</td><td>1</td><td>1</td><td>17421</td><td>110.8833</td><td>C68</td><td>C</td></tr><tr><td>586</td><td>1</td><td>1</td><td>Taussig, Miss. Ruth</td><td>female</td><td>18.0</td><td>0</td><td>2</td><td>110413</td><td>79.65</td><td>E68</td><td>S</td></tr><tr><td>592</td><td>1</td><td>1</td><td>Stephenson, Mrs. Walter Bertram (Martha Eustis)</td><td>female</td><td>52.0</td><td>1</td><td>0</td><td>36947</td><td>78.2667</td><td>D20</td><td>C</td></tr><tr><td>594</td><td>0</td><td>3</td><td>Bourke, Miss. Mary</td><td>female</td><td>null</td><td>0</td><td>2</td><td>364848</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>597</td><td>1</td><td>2</td><td>Leitch, Miss. Jessie Wills</td><td>female</td><td>null</td><td>0</td><td>0</td><td>248727</td><td>33.0</td><td>null</td><td>S</td></tr><tr><td>601</td><td>1</td><td>2</td><td>Jacobsohn, Mrs. Sidney Samuel (Amy Frances Christy)</td><td>female</td><td>24.0</td><td>2</td><td>1</td><td>243847</td><td>27.0</td><td>null</td><td>S</td></tr><tr><td>609</td><td>1</td><td>2</td><td>Laroche, Mrs. Joseph (Juliette Marie Louise Lafargue)</td><td>female</td><td>22.0</td><td>1</td><td>2</td><td>SC/Paris 2123</td><td>41.5792</td><td>null</td><td>C</td></tr><tr><td>610</td><td>1</td><td>1</td><td>Shutes, Miss. Elizabeth W</td><td>female</td><td>40.0</td><td>0</td><td>0</td><td>PC 17582</td><td>153.4625</td><td>C125</td><td>S</td></tr><tr><td>611</td><td>0</td><td>3</td><td>Andersson, Mrs. Anders Johan (Alfrida Konstantia Brogren)</td><td>female</td><td>39.0</td><td>1</td><td>5</td><td>347082</td><td>31.275</td><td>null</td><td>S</td></tr><tr><td>613</td><td>1</td><td>3</td><td>Murphy, Miss. Margaret Jane</td><td>female</td><td>null</td><td>1</td><td>0</td><td>367230</td><td>15.5</td><td>null</td><td>Q</td></tr><tr><td>616</td><td>1</td><td>2</td><td>Herman, Miss. Alice</td><td>female</td><td>24.0</td><td>1</td><td>2</td><td>220845</td><td>65.0</td><td>null</td><td>S</td></tr><tr><td>618</td><td>0</td><td>3</td><td>Lobb, Mrs. William Arthur (Cordelia K Stanlick)</td><td>female</td><td>26.0</td><td>1</td><td>0</td><td>A/5. 3336</td><td>16.1</td><td>null</td><td>S</td></tr><tr><td>619</td><td>1</td><td>2</td><td>Becker, Miss. Marion Louise</td><td>female</td><td>4.0</td><td>2</td><td>1</td><td>230136</td><td>39.0</td><td>F4</td><td>S</td></tr><tr><td>628</td><td>1</td><td>1</td><td>Longley, Miss. Gretchen Fiske</td><td>female</td><td>21.0</td><td>0</td><td>0</td><td>13502</td><td>77.9583</td><td>D9</td><td>S</td></tr><tr><td>635</td><td>0</td><td>3</td><td>Skoog, Miss. Mabel</td><td>female</td><td>9.0</td><td>3</td><td>2</td><td>347088</td><td>27.9</td><td>null</td><td>S</td></tr><tr><td>636</td><td>1</td><td>2</td><td>Davis, Miss. Mary</td><td>female</td><td>28.0</td><td>0</td><td>0</td><td>237668</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>639</td><td>0</td><td>3</td><td>Panula, Mrs. Juha (Maria Emilia Ojala)</td><td>female</td><td>41.0</td><td>0</td><td>5</td><td>3101295</td><td>39.6875</td><td>null</td><td>S</td></tr><tr><td>642</td><td>1</td><td>1</td><td>Sagesser, Mlle. Emma</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>PC 17477</td><td>69.3</td><td>B35</td><td>C</td></tr><tr><td>643</td><td>0</td><td>3</td><td>Skoog, Miss. Margit Elizabeth</td><td>female</td><td>2.0</td><td>3</td><td>2</td><td>347088</td><td>27.9</td><td>null</td><td>S</td></tr><tr><td>645</td><td>1</td><td>3</td><td>Baclini, Miss. Eugenie</td><td>female</td><td>0.75</td><td>2</td><td>1</td><td>2666</td><td>19.2583</td><td>null</td><td>C</td></tr><tr><td>650</td><td>1</td><td>3</td><td>Stanley, Miss. Amy Zillah Elsie</td><td>female</td><td>23.0</td><td>0</td><td>0</td><td>CA. 2314</td><td>7.55</td><td>null</td><td>S</td></tr><tr><td>652</td><td>1</td><td>2</td><td>Doling, Miss. Elsie</td><td>female</td><td>18.0</td><td>0</td><td>1</td><td>231919</td><td>23.0</td><td>null</td><td>S</td></tr><tr><td>654</td><td>1</td><td>3</td><td>\"O'Leary, Miss. Hanora \"\"Norah\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330919</td><td>7.8292</td><td>null</td><td>Q</td></tr><tr><td>655</td><td>0</td><td>3</td><td>\"Hegarty, Miss. Hanora \"\"Nora\"\"\"</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>365226</td><td>6.75</td><td>null</td><td>Q</td></tr><tr><td>658</td><td>0</td><td>3</td><td>Bourke, Mrs. John (Catherine)</td><td>female</td><td>32.0</td><td>1</td><td>1</td><td>364849</td><td>15.5</td><td>null</td><td>Q</td></tr><tr><td>670</td><td>1</td><td>1</td><td>Taylor, Mrs. Elmer Zebley (Juliet Cummins Wright)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>19996</td><td>52.0</td><td>C126</td><td>S</td></tr><tr><td>671</td><td>1</td><td>2</td><td>Brown, Mrs. Thomas William Solomon (Elizabeth Catherine Ford)</td><td>female</td><td>40.0</td><td>1</td><td>1</td><td>29750</td><td>39.0</td><td>null</td><td>S</td></tr><tr><td>678</td><td>1</td><td>3</td><td>Turja, Miss. Anna Sofia</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>4138</td><td>9.8417</td><td>null</td><td>S</td></tr><tr><td>679</td><td>0</td><td>3</td><td>Goodwin, Mrs. Frederick (Augusta Tyler)</td><td>female</td><td>43.0</td><td>1</td><td>6</td><td>CA 2144</td><td>46.9</td><td>null</td><td>S</td></tr><tr><td>681</td><td>0</td><td>3</td><td>Peters, Miss. Katie</td><td>female</td><td>null</td><td>0</td><td>0</td><td>330935</td><td>8.1375</td><td>null</td><td>Q</td></tr><tr><td>690</td><td>1</td><td>1</td><td>Madill, Miss. Georgette Alexandra</td><td>female</td><td>15.0</td><td>0</td><td>1</td><td>24160</td><td>211.3375</td><td>B5</td><td>S</td></tr><tr><td>692</td><td>1</td><td>3</td><td>Karun, Miss. Manca</td><td>female</td><td>4.0</td><td>0</td><td>1</td><td>349256</td><td>13.4167</td><td>null</td><td>C</td></tr><tr><td>698</td><td>1</td><td>3</td><td>\"Mullens, Miss. Katherine \"\"Katie\"\"\"</td><td>female</td><td>null</td><td>0</td><td>0</td><td>35852</td><td>7.7333</td><td>null</td><td>Q</td></tr><tr><td>701</td><td>1</td><td>1</td><td>Astor, Mrs. John Jacob (Madeleine Talmadge Force)</td><td>female</td><td>18.0</td><td>1</td><td>0</td><td>PC 17757</td><td>227.525</td><td>C62 C64</td><td>C</td></tr><tr><td>703</td><td>0</td><td>3</td><td>Barbara, Miss. Saiide</td><td>female</td><td>18.0</td><td>0</td><td>1</td><td>2691</td><td>14.4542</td><td>null</td><td>C</td></tr><tr><td>707</td><td>1</td><td>2</td><td>\"Kelly, Mrs. Florence \"\"Fannie\"\"\"</td><td>female</td><td>45.0</td><td>0</td><td>0</td><td>223596</td><td>13.5</td><td>null</td><td>S</td></tr><tr><td>709</td><td>1</td><td>1</td><td>Cleaver, Miss. Alice</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>113781</td><td>151.55</td><td>null</td><td>S</td></tr><tr><td>711</td><td>1</td><td>1</td><td>\"Mayne, Mlle. Berthe Antonine (\"\"Mrs de Villiers\"\")\"</td><td>female</td><td>24.0</td><td>0</td><td>0</td><td>PC 17482</td><td>49.5042</td><td>C90</td><td>C</td></tr><tr><td>717</td><td>1</td><td>1</td><td>Endres, Miss. Caroline Louise</td><td>female</td><td>38.0</td><td>0</td><td>0</td><td>PC 17757</td><td>227.525</td><td>C45</td><td>C</td></tr><tr><td>718</td><td>1</td><td>2</td><td>\"Troutt, Miss. Edwina Celia \"\"Winnie\"\"\"</td><td>female</td><td>27.0</td><td>0</td><td>0</td><td>34218</td><td>10.5</td><td>E101</td><td>S</td></tr><tr><td>721</td><td>1</td><td>2</td><td>\"Harper, Miss. Annie Jessie \"\"Nina\"\"\"</td><td>female</td><td>6.0</td><td>0</td><td>1</td><td>248727</td><td>33.0</td><td>null</td><td>S</td></tr><tr><td>727</td><td>1</td><td>2</td><td>Renouf, Mrs. Peter Henry (Lillian Jefferys)</td><td>female</td><td>30.0</td><td>3</td><td>0</td><td>31027</td><td>21.0</td><td>null</td><td>S</td></tr><tr><td>728</td><td>1</td><td>3</td><td>Mannion, Miss. Margareth</td><td>female</td><td>null</td><td>0</td><td>0</td><td>36866</td><td>7.7375</td><td>null</td><td>Q</td></tr><tr><td>730</td><td>0</td><td>3</td><td>Ilmakangas, Miss. Pieta Sofia</td><td>female</td><td>25.0</td><td>1</td><td>0</td><td>STON/O2. 3101271</td><td>7.925</td><td>null</td><td>S</td></tr><tr><td>731</td><td>1</td><td>1</td><td>Allen, Miss. Elisabeth Walton</td><td>female</td><td>29.0</td><td>0</td><td>0</td><td>24160</td><td>211.3375</td><td>B5</td><td>S</td></tr><tr><td>737</td><td>0</td><td>3</td><td>Ford, Mrs. Edward (Margaret Ann Watson)</td><td>female</td><td>48.0</td><td>1</td><td>3</td><td>W./C. 6608</td><td>34.375</td><td>null</td><td>S</td></tr><tr><td>743</td><td>1</td><td>1</td><td>\"Ryerson, Miss. Susan Parker \"\"Suzette\"\"\"</td><td>female</td><td>21.0</td><td>2</td><td>2</td><td>PC 17608</td><td>262.375</td><td>B57 B59 B63 B66</td><td>C</td></tr><tr><td>748</td><td>1</td><td>2</td><td>Sinkkonen, Miss. Anna</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>250648</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>751</td><td>1</td><td>2</td><td>Wells, Miss. Joan</td><td>female</td><td>4.0</td><td>1</td><td>1</td><td>29103</td><td>23.0</td><td>null</td><td>S</td></tr><tr><td>755</td><td>1</td><td>2</td><td>Herman, Mrs. Samuel (Jane Laver)</td><td>female</td><td>48.0</td><td>1</td><td>2</td><td>220845</td><td>65.0</td><td>null</td><td>S</td></tr><tr><td>760</td><td>1</td><td>1</td><td>Rothes, the Countess. of (Lucy Noel Martha Dyer-Edwards)</td><td>female</td><td>33.0</td><td>0</td><td>0</td><td>110152</td><td>86.5</td><td>B77</td><td>S</td></tr><tr><td>764</td><td>1</td><td>1</td><td>Carter, Mrs. William Ernest (Lucile Polk)</td><td>female</td><td>36.0</td><td>1</td><td>2</td><td>113760</td><td>120.0</td><td>B96 B98</td><td>S</td></tr><tr><td>766</td><td>1</td><td>1</td><td>Hogeboom, Mrs. John C (Anna Andrews)</td><td>female</td><td>51.0</td><td>1</td><td>0</td><td>13502</td><td>77.9583</td><td>D11</td><td>S</td></tr><tr><td>768</td><td>0</td><td>3</td><td>Mangan, Miss. Mary</td><td>female</td><td>30.5</td><td>0</td><td>0</td><td>364850</td><td>7.75</td><td>null</td><td>Q</td></tr><tr><td>773</td><td>0</td><td>2</td><td>Mack, Mrs. (Mary)</td><td>female</td><td>57.0</td><td>0</td><td>0</td><td>S.O./P.P. 3</td><td>10.5</td><td>E77</td><td>S</td></tr><tr><td>775</td><td>1</td><td>2</td><td>Hocking, Mrs. Elizabeth (Eliza Needs)</td><td>female</td><td>54.0</td><td>1</td><td>3</td><td>29105</td><td>23.0</td><td>null</td><td>S</td></tr><tr><td>778</td><td>1</td><td>3</td><td>Emanuel, Miss. Virginia Ethel</td><td>female</td><td>5.0</td><td>0</td><td>0</td><td>364516</td><td>12.475</td><td>null</td><td>S</td></tr><tr><td>780</td><td>1</td><td>1</td><td>Robert, Mrs. Edward Scott (Elisabeth Walton McMillan)</td><td>female</td><td>43.0</td><td>0</td><td>1</td><td>24160</td><td>211.3375</td><td>B3</td><td>S</td></tr><tr><td>781</td><td>1</td><td>3</td><td>Ayoub, Miss. Banoura</td><td>female</td><td>13.0</td><td>0</td><td>0</td><td>2687</td><td>7.2292</td><td>null</td><td>C</td></tr><tr><td>782</td><td>1</td><td>1</td><td>Dick, Mrs. Albert Adrian (Vera Gillespie)</td><td>female</td><td>17.0</td><td>1</td><td>0</td><td>17474</td><td>57.0</td><td>B20</td><td>S</td></tr><tr><td>787</td><td>1</td><td>3</td><td>Sjoblom, Miss. Anna Sofia</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>3101265</td><td>7.4958</td><td>null</td><td>S</td></tr><tr><td>793</td><td>0</td><td>3</td><td>Sage, Miss. Stella Anna</td><td>female</td><td>null</td><td>8</td><td>2</td><td>CA. 2343</td><td>69.55</td><td>null</td><td>S</td></tr><tr><td>797</td><td>1</td><td>1</td><td>Leader, Dr. Alice (Farnham)</td><td>female</td><td>49.0</td><td>0</td><td>0</td><td>17465</td><td>25.9292</td><td>D17</td><td>S</td></tr><tr><td>798</td><td>1</td><td>3</td><td>Osman, Mrs. Mara</td><td>female</td><td>31.0</td><td>0</td><td>0</td><td>349244</td><td>8.6833</td><td>null</td><td>S</td></tr><tr><td>800</td><td>0</td><td>3</td><td>Van Impe, Mrs. Jean Baptiste (Rosalie Paula Govaert)</td><td>female</td><td>30.0</td><td>1</td><td>1</td><td>345773</td><td>24.15</td><td>null</td><td>S</td></tr><tr><td>802</td><td>1</td><td>2</td><td>Collyer, Mrs. Harvey (Charlotte Annie Tate)</td><td>female</td><td>31.0</td><td>1</td><td>1</td><td>C.A. 31921</td><td>26.25</td><td>null</td><td>S</td></tr><tr><td>808</td><td>0</td><td>3</td><td>Pettersson, Miss. Ellen Natalia</td><td>female</td><td>18.0</td><td>0</td><td>0</td><td>347087</td><td>7.775</td><td>null</td><td>S</td></tr><tr><td>810</td><td>1</td><td>1</td><td>Chambers, Mrs. Norman Campbell (Bertha Griggs)</td><td>female</td><td>33.0</td><td>1</td><td>0</td><td>113806</td><td>53.1</td><td>E8</td><td>S</td></tr><tr><td>814</td><td>0</td><td>3</td><td>Andersson, Miss. Ebba Iris Alfrida</td><td>female</td><td>6.0</td><td>4</td><td>2</td><td>347082</td><td>31.275</td><td>null</td><td>S</td></tr><tr><td>817</td><td>0</td><td>3</td><td>Heininen, Miss. Wendla Maria</td><td>female</td><td>23.0</td><td>0</td><td>0</td><td>STON/O2. 3101290</td><td>7.925</td><td>null</td><td>S</td></tr><tr><td>821</td><td>1</td><td>1</td><td>Hays, Mrs. Charles Melville (Clara Jennings Gregg)</td><td>female</td><td>52.0</td><td>1</td><td>1</td><td>12749</td><td>93.5</td><td>B69</td><td>S</td></tr><tr><td>824</td><td>1</td><td>3</td><td>Moor, Mrs. (Beila)</td><td>female</td><td>27.0</td><td>0</td><td>1</td><td>392096</td><td>12.475</td><td>E121</td><td>S</td></tr><tr><td>830</td><td>1</td><td>1</td><td>Stone, Mrs. George Nelson (Martha Evelyn)</td><td>female</td><td>62.0</td><td>0</td><td>0</td><td>113572</td><td>80.0</td><td>B28</td><td>null</td></tr><tr><td>831</td><td>1</td><td>3</td><td>Yasbeck, Mrs. Antoni (Selini Alexander)</td><td>female</td><td>15.0</td><td>1</td><td>0</td><td>2659</td><td>14.4542</td><td>null</td><td>C</td></tr><tr><td>836</td><td>1</td><td>1</td><td>Compton, Miss. Sara Rebecca</td><td>female</td><td>39.0</td><td>1</td><td>1</td><td>PC 17756</td><td>83.1583</td><td>E49</td><td>C</td></tr><tr><td>843</td><td>1</td><td>1</td><td>Serepeca, Miss. Augusta</td><td>female</td><td>30.0</td><td>0</td><td>0</td><td>113798</td><td>31.0</td><td>null</td><td>C</td></tr><tr><td>850</td><td>1</td><td>1</td><td>Goldenberg, Mrs. Samuel L (Edwiga Grabowska)</td><td>female</td><td>null</td><td>1</td><td>0</td><td>17453</td><td>89.1042</td><td>C92</td><td>C</td></tr><tr><td>853</td><td>0</td><td>3</td><td>Boulos, Miss. Nourelain</td><td>female</td><td>9.0</td><td>1</td><td>1</td><td>2678</td><td>15.2458</td><td>null</td><td>C</td></tr><tr><td>854</td><td>1</td><td>1</td><td>Lines, Miss. Mary Conover</td><td>female</td><td>16.0</td><td>0</td><td>1</td><td>PC 17592</td><td>39.4</td><td>D28</td><td>S</td></tr><tr><td>855</td><td>0</td><td>2</td><td>Carter, Mrs. Ernest Courtenay (Lilian Hughes)</td><td>female</td><td>44.0</td><td>1</td><td>0</td><td>244252</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>856</td><td>1</td><td>3</td><td>Aks, Mrs. Sam (Leah Rosen)</td><td>female</td><td>18.0</td><td>0</td><td>1</td><td>392091</td><td>9.35</td><td>null</td><td>S</td></tr><tr><td>857</td><td>1</td><td>1</td><td>Wick, Mrs. George Dennick (Mary Hitchcock)</td><td>female</td><td>45.0</td><td>1</td><td>1</td><td>36928</td><td>164.8667</td><td>null</td><td>S</td></tr><tr><td>859</td><td>1</td><td>3</td><td>Baclini, Mrs. Solomon (Latifa Qurban)</td><td>female</td><td>24.0</td><td>0</td><td>3</td><td>2666</td><td>19.2583</td><td>null</td><td>C</td></tr><tr><td>863</td><td>1</td><td>1</td><td>Swift, Mrs. Frederick Joel (Margaret Welles Barron)</td><td>female</td><td>48.0</td><td>0</td><td>0</td><td>17466</td><td>25.9292</td><td>D17</td><td>S</td></tr><tr><td>864</td><td>0</td><td>3</td><td>\"Sage, Miss. Dorothy Edith \"\"Dolly\"\"\"</td><td>female</td><td>null</td><td>8</td><td>2</td><td>CA. 2343</td><td>69.55</td><td>null</td><td>S</td></tr><tr><td>866</td><td>1</td><td>2</td><td>Bystrom, Mrs. (Karolina)</td><td>female</td><td>42.0</td><td>0</td><td>0</td><td>236852</td><td>13.0</td><td>null</td><td>S</td></tr><tr><td>867</td><td>1</td><td>2</td><td>Duran y More, Miss. Asuncion</td><td>female</td><td>27.0</td><td>1</td><td>0</td><td>SC/PARIS 2149</td><td>13.8583</td><td>null</td><td>C</td></tr><tr><td>872</td><td>1</td><td>1</td><td>Beckwith, Mrs. Richard Leonard (Sallie Monypeny)</td><td>female</td><td>47.0</td><td>1</td><td>1</td><td>11751</td><td>52.5542</td><td>D35</td><td>S</td></tr><tr><td>875</td><td>1</td><td>2</td><td>Abelson, Mrs. Samuel (Hannah Wizosky)</td><td>female</td><td>28.0</td><td>1</td><td>0</td><td>P/PP 3381</td><td>24.0</td><td>null</td><td>C</td></tr><tr><td>876</td><td>1</td><td>3</td><td>\"Najib, Miss. Adele Kiamie \"\"Jane\"\"\"</td><td>female</td><td>15.0</td><td>0</td><td>0</td><td>2667</td><td>7.225</td><td>null</td><td>C</td></tr><tr><td>880</td><td>1</td><td>1</td><td>Potter, Mrs. Thomas Jr (Lily Alexenia Wilson)</td><td>female</td><td>56.0</td><td>0</td><td>1</td><td>11767</td><td>83.1583</td><td>C50</td><td>C</td></tr><tr><td>881</td><td>1</td><td>2</td><td>Shelley, Mrs. William (Imanita Parrish Hall)</td><td>female</td><td>25.0</td><td>0</td><td>1</td><td>230433</td><td>26.0</td><td>null</td><td>S</td></tr><tr><td>883</td><td>0</td><td>3</td><td>Dahlberg, Miss. Gerda Ulrika</td><td>female</td><td>22.0</td><td>0</td><td>0</td><td>7552</td><td>10.5167</td><td>null</td><td>S</td></tr><tr><td>886</td><td>0</td><td>3</td><td>Rice, Mrs. William (Margaret Norton)</td><td>female</td><td>39.0</td><td>0</td><td>5</td><td>382652</td><td>29.125</td><td>null</td><td>Q</td></tr><tr><td>888</td><td>1</td><td>1</td><td>Graham, Miss. Margaret Edith</td><td>female</td><td>19.0</td><td>0</td><td>0</td><td>112053</td><td>30.0</td><td>B42</td><td>S</td></tr><tr><td>889</td><td>0</td><td>3</td><td>\"Johnston, Miss. Catherine Helen \"\"Carrie\"\"\"</td><td>female</td><td>null</td><td>1</td><td>2</td><td>W./C. 6607</td><td>23.45</td><td>null</td><td>S</td></tr></tbody></table></div>"]}}],"execution_count":13},{"cell_type":"code","source":["spark.catalog.listTables()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">47</span><span class=\"ansired\">]: </span>\n[Table(name=&apos;cs_test_31b97_csv&apos;, database=&apos;default&apos;, description=None, tableType=&apos;EXTERNAL&apos;, isTemporary=False),\n Table(name=&apos;hr_comma_sep_csv_55265_txt&apos;, database=&apos;default&apos;, description=None, tableType=&apos;EXTERNAL&apos;, isTemporary=False),\n Table(name=&apos;hr_data&apos;, database=&apos;default&apos;, description=None, tableType=&apos;EXTERNAL&apos;, isTemporary=False),\n Table(name=&apos;titanic&apos;, database=None, description=None, tableType=&apos;TEMPORARY&apos;, isTemporary=True)]\n</div>"]}}],"execution_count":14},{"cell_type":"code","source":["spark.catalog.dropTempView('titanic')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":15},{"cell_type":"code","source":["titanic_data.createOrReplaceGlobalTempView('titanic_global')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":16},{"cell_type":"code","source":["spark.catalog.dropGlobalTempView('titanic_globalspar')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":17},{"cell_type":"code","source":["spark.sql(\"select * from global_titanic_global\")"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">AnalysisException</span>                         Traceback (most recent call last)\n<span class=\"ansigreen\">&lt;command-519762779959754&gt;</span> in <span class=\"ansicyan\">&lt;module&gt;</span><span class=\"ansiblue\">()</span>\n<span class=\"ansigreen\">----&gt; 1</span><span class=\"ansiyellow\"> </span>spark<span class=\"ansiyellow\">.</span>sql<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&quot;select * from global_titanic_global&quot;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/session.py</span> in <span class=\"ansicyan\">sql</span><span class=\"ansiblue\">(self, sqlQuery)</span>\n<span class=\"ansigreen\">    827</span>         <span class=\"ansiyellow\">[</span>Row<span class=\"ansiyellow\">(</span>f1<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">,</span> f2<span class=\"ansiyellow\">=</span><span class=\"ansiblue\">u&apos;row1&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">,</span> Row<span class=\"ansiyellow\">(</span>f1<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">2</span><span class=\"ansiyellow\">,</span> f2<span class=\"ansiyellow\">=</span><span class=\"ansiblue\">u&apos;row2&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">,</span> Row<span class=\"ansiyellow\">(</span>f1<span class=\"ansiyellow\">=</span><span class=\"ansicyan\">3</span><span class=\"ansiyellow\">,</span> f2<span class=\"ansiyellow\">=</span><span class=\"ansiblue\">u&apos;row3&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">]</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    828</span>         &quot;&quot;&quot;\n<span class=\"ansigreen\">--&gt; 829</span><span class=\"ansiyellow\">         </span><span class=\"ansigreen\">return</span> DataFrame<span class=\"ansiyellow\">(</span>self<span class=\"ansiyellow\">.</span>_jsparkSession<span class=\"ansiyellow\">.</span>sql<span class=\"ansiyellow\">(</span>sqlQuery<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">,</span> self<span class=\"ansiyellow\">.</span>_wrapped<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    830</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    831</span>     <span class=\"ansiyellow\">@</span>since<span class=\"ansiyellow\">(</span><span class=\"ansicyan\">2.0</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py</span> in <span class=\"ansicyan\">__call__</span><span class=\"ansiblue\">(self, *args)</span>\n<span class=\"ansigreen\">   1255</span>         answer <span class=\"ansiyellow\">=</span> self<span class=\"ansiyellow\">.</span>gateway_client<span class=\"ansiyellow\">.</span>send_command<span class=\"ansiyellow\">(</span>command<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1256</span>         return_value = get_return_value(\n<span class=\"ansigreen\">-&gt; 1257</span><span class=\"ansiyellow\">             answer, self.gateway_client, self.target_id, self.name)\n</span><span class=\"ansigreen\">   1258</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1259</span>         <span class=\"ansigreen\">for</span> temp_arg <span class=\"ansigreen\">in</span> temp_args<span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/utils.py</span> in <span class=\"ansicyan\">deco</span><span class=\"ansiblue\">(*a, **kw)</span>\n<span class=\"ansigreen\">     67</span>                                              e.java_exception.getStackTrace()))\n<span class=\"ansigreen\">     68</span>             <span class=\"ansigreen\">if</span> s<span class=\"ansiyellow\">.</span>startswith<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;org.apache.spark.sql.AnalysisException: &apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 69</span><span class=\"ansiyellow\">                 </span><span class=\"ansigreen\">raise</span> AnalysisException<span class=\"ansiyellow\">(</span>s<span class=\"ansiyellow\">.</span>split<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;: &apos;</span><span class=\"ansiyellow\">,</span> <span class=\"ansicyan\">1</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">[</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">]</span><span class=\"ansiyellow\">,</span> stackTrace<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     70</span>             <span class=\"ansigreen\">if</span> s<span class=\"ansiyellow\">.</span>startswith<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;org.apache.spark.sql.catalyst.analysis&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     71</span>                 <span class=\"ansigreen\">raise</span> AnalysisException<span class=\"ansiyellow\">(</span>s<span class=\"ansiyellow\">.</span>split<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;: &apos;</span><span class=\"ansiyellow\">,</span> <span class=\"ansicyan\">1</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">[</span><span class=\"ansicyan\">1</span><span class=\"ansiyellow\">]</span><span class=\"ansiyellow\">,</span> stackTrace<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansired\">AnalysisException</span>: &apos;Table or view not found: global_titanic_global; line 1 pos 14&apos;</div>"]}}],"execution_count":18},{"cell_type":"code","source":["stringCSVRDD = sc.parallelize([(123, 'Katie', 19, 'brown'), (234, 'Michael', 22, 'green'), (345, 'Simone', 22, 11)])\n\nfrom pyspark.sql.types import StringType, StructField, LongType, StructType"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":19},{"cell_type":"code","source":["schema = StructType([\n    StructField(\"id\", LongType(), True),    \n    StructField(\"name\", StringType(), True),\n    StructField(\"age\", LongType(), True),\n    StructField(\"eyeColor\", StringType(), True)\n])"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":20},{"cell_type":"code","source":["df = spark.createDataFrame(stringCSVRDD, schema)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":21},{"cell_type":"code","source":["df[df.id == 345].count()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">84</span><span class=\"ansired\">]: </span>1\n</div>"]}}],"execution_count":22},{"cell_type":"code","source":["from pyspark.sql.types import Row"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":23},{"cell_type":"code","source":["# Create the Departments\ndepartment1 = Row(id='123456', name='Computer Science')\ndepartment2 = Row(id='789012', name='Mechanical Engineering')\ndepartment3 = Row(id='345678', name='Theater and Drama')\ndepartment4 = Row(id='901234', name='Indoor Recreation')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":24},{"cell_type":"code","source":["Employee = Row(\"firstName\", \"lastName\", \"email\", \"salary\")"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":25},{"cell_type":"code","source":["employee1 = Employee('michael', 'armbrust', 'no-reply@berkeley.edu', 100000)\nemployee11 = Employee('michael', 'armbrust', 'no-reply@berkeley.edu', 200000)\nemployee2 = Employee('xiangrui', 'meng', 'no-reply@stanford.edu', 120000)\nemployee3 = Employee('matei', None, 'no-reply@waterloo.edu', 140000)\nemployee31 = Employee('matei', None, 'no-reply@waterloo.edu', 180000)\nemployee4 = Employee(None, 'wendell', 'no-reply@berkeley.edu', 160000)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":26},{"cell_type":"code","source":["departmentWithEmployees1 = Row(department=department1, employees=[employee1, employee2])\ndepartmentWithEmployees2 = Row(department=department2, employees=[employee3, employee4, employee11])\ndepartmentWithEmployees3 = Row(department=department3, employees=[employee1, employee4, employee31])\ndepartmentWithEmployees4 = Row(department=department4, employees=[employee2, employee3])"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":27},{"cell_type":"code","source":["df = spark.createDataFrame([departmentWithEmployees1,departmentWithEmployees2,departmentWithEmployees3,departmentWithEmployees4])"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":28},{"cell_type":"code","source":["df.write.json('emp.csv')"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">---------------------------------------------------------------------------</span>\n<span class=\"ansired\">Py4JJavaError</span>                             Traceback (most recent call last)\n<span class=\"ansigreen\">&lt;command-519762779959765&gt;</span> in <span class=\"ansicyan\">&lt;module&gt;</span><span class=\"ansiblue\">()</span>\n<span class=\"ansigreen\">----&gt; 1</span><span class=\"ansiyellow\"> </span>df<span class=\"ansiyellow\">.</span>write<span class=\"ansiyellow\">.</span>json<span class=\"ansiyellow\">(</span><span class=\"ansiblue\">&apos;emp.csv&apos;</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/readwriter.py</span> in <span class=\"ansicyan\">json</span><span class=\"ansiblue\">(self, path, mode, compression, dateFormat, timestampFormat, lineSep, encoding)</span>\n<span class=\"ansigreen\">    812</span>             compression<span class=\"ansiyellow\">=</span>compression<span class=\"ansiyellow\">,</span> dateFormat<span class=\"ansiyellow\">=</span>dateFormat<span class=\"ansiyellow\">,</span> timestampFormat<span class=\"ansiyellow\">=</span>timestampFormat<span class=\"ansiyellow\">,</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    813</span>             lineSep=lineSep, encoding=encoding)\n<span class=\"ansigreen\">--&gt; 814</span><span class=\"ansiyellow\">         </span>self<span class=\"ansiyellow\">.</span>_jwrite<span class=\"ansiyellow\">.</span>json<span class=\"ansiyellow\">(</span>path<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    815</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    816</span>     <span class=\"ansiyellow\">@</span>since<span class=\"ansiyellow\">(</span><span class=\"ansicyan\">1.4</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py</span> in <span class=\"ansicyan\">__call__</span><span class=\"ansiblue\">(self, *args)</span>\n<span class=\"ansigreen\">   1255</span>         answer <span class=\"ansiyellow\">=</span> self<span class=\"ansiyellow\">.</span>gateway_client<span class=\"ansiyellow\">.</span>send_command<span class=\"ansiyellow\">(</span>command<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1256</span>         return_value = get_return_value(\n<span class=\"ansigreen\">-&gt; 1257</span><span class=\"ansiyellow\">             answer, self.gateway_client, self.target_id, self.name)\n</span><span class=\"ansigreen\">   1258</span> <span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">   1259</span>         <span class=\"ansigreen\">for</span> temp_arg <span class=\"ansigreen\">in</span> temp_args<span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/pyspark/sql/utils.py</span> in <span class=\"ansicyan\">deco</span><span class=\"ansiblue\">(*a, **kw)</span>\n<span class=\"ansigreen\">     61</span>     <span class=\"ansigreen\">def</span> deco<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     62</span>         <span class=\"ansigreen\">try</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">---&gt; 63</span><span class=\"ansiyellow\">             </span><span class=\"ansigreen\">return</span> f<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">*</span>a<span class=\"ansiyellow\">,</span> <span class=\"ansiyellow\">**</span>kw<span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     64</span>         <span class=\"ansigreen\">except</span> py4j<span class=\"ansiyellow\">.</span>protocol<span class=\"ansiyellow\">.</span>Py4JJavaError <span class=\"ansigreen\">as</span> e<span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">     65</span>             s <span class=\"ansiyellow\">=</span> e<span class=\"ansiyellow\">.</span>java_exception<span class=\"ansiyellow\">.</span>toString<span class=\"ansiyellow\">(</span><span class=\"ansiyellow\">)</span><span class=\"ansiyellow\"></span>\n\n<span class=\"ansigreen\">/databricks/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py</span> in <span class=\"ansicyan\">get_return_value</span><span class=\"ansiblue\">(answer, gateway_client, target_id, name)</span>\n<span class=\"ansigreen\">    326</span>                 raise Py4JJavaError(\n<span class=\"ansigreen\">    327</span>                     <span class=\"ansiblue\">&quot;An error occurred while calling {0}{1}{2}.\\n&quot;</span><span class=\"ansiyellow\">.</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">--&gt; 328</span><span class=\"ansiyellow\">                     format(target_id, &quot;.&quot;, name), value)\n</span><span class=\"ansigreen\">    329</span>             <span class=\"ansigreen\">else</span><span class=\"ansiyellow\">:</span><span class=\"ansiyellow\"></span>\n<span class=\"ansigreen\">    330</span>                 raise Py4JError(\n\n<span class=\"ansired\">Py4JJavaError</span>: An error occurred while calling o3985.json.\n: org.apache.spark.SparkException: Job aborted.\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:196)\n\tat org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:192)\n\tat org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:110)\n\tat org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:108)\n\tat org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:128)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:143)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)\n\tat org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$5.apply(SparkPlan.scala:183)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:180)\n\tat org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:131)\n\tat org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:114)\n\tat org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:114)\n\tat org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:690)\n\tat org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:690)\n\tat org.apache.spark.sql.execution.SQLExecution$$anonfun$withCustomExecutionEnv$1.apply(SQLExecution.scala:99)\n\tat org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:228)\n\tat org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:85)\n\tat org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:158)\n\tat org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:690)\n\tat org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:290)\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:284)\n\tat org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:232)\n\tat org.apache.spark.sql.DataFrameWriter.json(DataFrameWriter.scala:558)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:498)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:380)\n\tat py4j.Gateway.invoke(Gateway.java:295)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:251)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 5 in stage 155.0 failed 1 times, most recent failure: Lost task 5.0 in stage 155.0 (TID 420, localhost, executor driver): java.rmi.RemoteException: com.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293); nested exception is: \n\tcom.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:100)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.sendIgnoreDraining(DbfsClient.scala:69)\n\tat com.databricks.backend.daemon.data.client.DbfsOutputStream.close0(DbfsOutputStream.scala:99)\n\tat com.databricks.backend.daemon.data.client.DbfsOutputStream.close(DbfsOutputStream.scala:78)\n\tat java.io.FilterOutputStream.close(FilterOutputStream.java:159)\n\tat org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)\n\tat org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)\n\tat com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol.newTaskTempFileAbsPath(DirectoryAtomicCommitProtocol.scala:99)\n\tat com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol.newTaskTempFile(DirectoryAtomicCommitProtocol.scala:84)\n\tat org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.newOutputWriter(FileFormatDataWriter.scala:115)\n\tat org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.&lt;init&gt;(FileFormatDataWriter.scala:108)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:233)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:169)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:168)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.doRunTask(Task.scala:139)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:112)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$13.apply(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1432)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:503)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\tat java.lang.Thread.run(Thread.java:748)\nCaused by: com.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293)\n\tat com.databricks.api.rpc.ScalaProtoRpcSerializer.deserializeException(ScalaProtoRpcSerializer.scala:161)\n\tat com.databricks.api.rpc.ScalaProtoRpcSerializer.deserializeException(ScalaProtoRpcSerializer.scala:153)\n\tat com.databricks.rpc.JettyClient.$anonfun$getResponseFromExchange$3(JettyClient.scala:324)\n\tat com.databricks.util.UntrustedUtils$.logUncaughtExceptions(UntrustedUtils.scala:34)\n\tat com.databricks.rpc.JettyClient.getResponseFromExchange(JettyClient.scala:322)\n\tat com.databricks.rpc.JettyClient.processResponse0(JettyClient.scala:357)\n\tat com.databricks.rpc.JettyClient.$anonfun$processResponse$1(JettyClient.scala:346)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.JettyClient.withAttributionContext(JettyClient.scala:32)\n\tat com.databricks.rpc.JettyClient.com$databricks$rpc$JettyClient$$processResponse(JettyClient.scala:345)\n\tat com.databricks.rpc.JettyClient.$anonfun$send$1(JettyClient.scala:74)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.rpc.JettyClient.send(JettyClient.scala:74)\n\tat com.databricks.rpc.BoundRPCClient.send(BoundRPCClient.scala:36)\n\tat com.databricks.rpc.ReliableJettyClient.$anonfun$sendNonIdempotent$2(ReliableJettyClient.scala:83)\n\tat com.databricks.rpc.ReliableJettyClient.retryOnTransientError(ReliableJettyClient.scala:174)\n\tat com.databricks.rpc.ReliableJettyClient.sendNonIdempotent(ReliableJettyClient.scala:83)\n\tat com.databricks.backend.daemon.data.server.DbfsLimitEnforcer.allocate(DbfsLimitEnforcer.scala:22)\n\tat com.databricks.s3a.enforcer.S3AEnforcer$.allocateWithOverwriteCheck(S3AEnforcer.scala:43)\n\tat com.databricks.s3a.aws.transfer.EnforcingDatabricksTransferManager.upload(EnforcingDatabricksTransferManager.scala:36)\n\tat com.databricks.s3a.S3AOutputStream$1.apply(S3AOutputStream.java:144)\n\tat com.databricks.s3a.S3AOutputStream$1.apply(S3AOutputStream.java:139)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.s3a.aws.DatabricksS3Client.retryRequest(DatabricksS3Client.scala:145)\n\tat com.databricks.s3a.aws.DatabricksS3Client.withExponentialBackoff(DatabricksS3Client.scala:119)\n\tat com.databricks.s3a.aws.DatabricksS3Client.withExponentialBackoff$(DatabricksS3Client.scala:115)\n\tat com.databricks.s3a.aws.EnforcingDatabricksS3Client.withExponentialBackoff(EnforcingDatabricksS3Client.scala:25)\n\tat com.databricks.s3a.S3AOutputStream.close(S3AOutputStream.java:139)\n\tat org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)\n\tat org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)\n\tat java.io.FilterOutputStream.close(FilterOutputStream.java:159)\n\tat com.databricks.backend.daemon.data.server.session.IOStreamManager$OutStream.close(IOStreamManager.scala:155)\n\tat com.databricks.backend.daemon.data.server.handler.IOStreamHandler.receive(IOStreamHandler.scala:25)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.$anonfun$queryHandlers$1(SessionContext.scala:83)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.$anonfun$queryHandlers$1$adapted(SessionContext.scala:82)\n\tat scala.collection.immutable.List.foreach(List.scala:392)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.queryHandlers(SessionContext.scala:82)\n\tat com.databricks.backend.daemon.data.server.DbfsServerBackend$$anonfun$receive$3.applyOrElse(DbfsServerBackend.scala:272)\n\tat com.databricks.backend.daemon.data.server.DbfsServerBackend$$anonfun$receive$3.applyOrElse(DbfsServerBackend.scala:252)\n\tat com.databricks.rpc.ServerBackend.$anonfun$internalReceive$2(ServerBackend.scala:42)\n\tat com.databricks.rpc.ServerBackend$$anonfun$commonReceive$1.applyOrElse(ServerBackend.scala:58)\n\tat com.databricks.rpc.ServerBackend$$anonfun$commonReceive$1.applyOrElse(ServerBackend.scala:58)\n\tat com.databricks.rpc.ServerBackend.$anonfun$internalReceive$1(ServerBackend.scala:38)\n\tat com.databricks.logging.UsageLogging.$anonfun$recordOperation$2(UsageLogging.scala:359)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.ServerBackend.withAttributionContext(ServerBackend.scala:13)\n\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:268)\n\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:264)\n\tat com.databricks.rpc.ServerBackend.withAttributionTags(ServerBackend.scala:13)\n\tat com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:345)\n\tat com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:320)\n\tat com.databricks.rpc.ServerBackend.recordOperation(ServerBackend.scala:13)\n\tat com.databricks.rpc.ServerBackend.internalReceive(ServerBackend.scala:38)\n\tat com.databricks.rpc.JettyServer$RequestManager.$anonfun$handleRPC$2(JettyServer.scala:346)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleRPC(JettyServer.scala:346)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleRequestAndRespond(JettyServer.scala:269)\n\tat com.databricks.rpc.JettyServer$RequestManager.$anonfun$handleHttp$4(JettyServer.scala:180)\n\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.JettyServer$.withAttributionContext(JettyServer.scala:106)\n\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:268)\n\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:264)\n\tat com.databricks.rpc.JettyServer$.withAttributionTags(JettyServer.scala:106)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleHttp(JettyServer.scala:171)\n\tat com.databricks.rpc.JettyServer$RequestManager.doPost(JettyServer.scala:133)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:707)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n\tat org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:848)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:539)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\t... 1 more\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:2100)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2088)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:2087)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2087)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:1076)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:1076)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1076)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2319)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2267)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2255)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:873)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:2252)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:166)\n\t... 34 more\nCaused by: java.rmi.RemoteException: com.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293); nested exception is: \n\tcom.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.send0(DbfsClient.scala:100)\n\tat com.databricks.backend.daemon.data.client.DbfsClient.sendIgnoreDraining(DbfsClient.scala:69)\n\tat com.databricks.backend.daemon.data.client.DbfsOutputStream.close0(DbfsOutputStream.scala:99)\n\tat com.databricks.backend.daemon.data.client.DbfsOutputStream.close(DbfsOutputStream.scala:78)\n\tat java.io.FilterOutputStream.close(FilterOutputStream.java:159)\n\tat org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)\n\tat org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)\n\tat com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol.newTaskTempFileAbsPath(DirectoryAtomicCommitProtocol.scala:99)\n\tat com.databricks.sql.transaction.directory.DirectoryAtomicCommitProtocol.newTaskTempFile(DirectoryAtomicCommitProtocol.scala:84)\n\tat org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.newOutputWriter(FileFormatDataWriter.scala:115)\n\tat org.apache.spark.sql.execution.datasources.SingleDirectoryDataWriter.&lt;init&gt;(FileFormatDataWriter.scala:108)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$.org$apache$spark$sql$execution$datasources$FileFormatWriter$$executeTask(FileFormatWriter.scala:233)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:169)\n\tat org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:168)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)\n\tat org.apache.spark.scheduler.Task.doRunTask(Task.scala:139)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:112)\n\tat org.apache.spark.executor.Executor$TaskRunner$$anonfun$13.apply(Executor.scala:497)\n\tat org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1432)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:503)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n\t... 1 more\nCaused by: com.databricks.api.base.DatabricksServiceException: QUOTA_EXCEEDED: You have exceeded the maximum number of allowed files on Databricks Community Edition. To ensure free access, you are limited to 10000 files and 10 GB of storage in DBFS. Please use dbutils.fs to list and clean up files to restore service. You may have to wait a few minutes after cleaning up the files for the quota to be refreshed. (Files found: 16293)\n\tat com.databricks.api.rpc.ScalaProtoRpcSerializer.deserializeException(ScalaProtoRpcSerializer.scala:161)\n\tat com.databricks.api.rpc.ScalaProtoRpcSerializer.deserializeException(ScalaProtoRpcSerializer.scala:153)\n\tat com.databricks.rpc.JettyClient.$anonfun$getResponseFromExchange$3(JettyClient.scala:324)\n\tat com.databricks.util.UntrustedUtils$.logUncaughtExceptions(UntrustedUtils.scala:34)\n\tat com.databricks.rpc.JettyClient.getResponseFromExchange(JettyClient.scala:322)\n\tat com.databricks.rpc.JettyClient.processResponse0(JettyClient.scala:357)\n\tat com.databricks.rpc.JettyClient.$anonfun$processResponse$1(JettyClient.scala:346)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.JettyClient.withAttributionContext(JettyClient.scala:32)\n\tat com.databricks.rpc.JettyClient.com$databricks$rpc$JettyClient$$processResponse(JettyClient.scala:345)\n\tat com.databricks.rpc.JettyClient.$anonfun$send$1(JettyClient.scala:74)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.rpc.JettyClient.send(JettyClient.scala:74)\n\tat com.databricks.rpc.BoundRPCClient.send(BoundRPCClient.scala:36)\n\tat com.databricks.rpc.ReliableJettyClient.$anonfun$sendNonIdempotent$2(ReliableJettyClient.scala:83)\n\tat com.databricks.rpc.ReliableJettyClient.retryOnTransientError(ReliableJettyClient.scala:174)\n\tat com.databricks.rpc.ReliableJettyClient.sendNonIdempotent(ReliableJettyClient.scala:83)\n\tat com.databricks.backend.daemon.data.server.DbfsLimitEnforcer.allocate(DbfsLimitEnforcer.scala:22)\n\tat com.databricks.s3a.enforcer.S3AEnforcer$.allocateWithOverwriteCheck(S3AEnforcer.scala:43)\n\tat com.databricks.s3a.aws.transfer.EnforcingDatabricksTransferManager.upload(EnforcingDatabricksTransferManager.scala:36)\n\tat com.databricks.s3a.S3AOutputStream$1.apply(S3AOutputStream.java:144)\n\tat com.databricks.s3a.S3AOutputStream$1.apply(S3AOutputStream.java:139)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.s3a.aws.DatabricksS3Client.retryRequest(DatabricksS3Client.scala:145)\n\tat com.databricks.s3a.aws.DatabricksS3Client.withExponentialBackoff(DatabricksS3Client.scala:119)\n\tat com.databricks.s3a.aws.DatabricksS3Client.withExponentialBackoff$(DatabricksS3Client.scala:115)\n\tat com.databricks.s3a.aws.EnforcingDatabricksS3Client.withExponentialBackoff(EnforcingDatabricksS3Client.scala:25)\n\tat com.databricks.s3a.S3AOutputStream.close(S3AOutputStream.java:139)\n\tat org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72)\n\tat org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106)\n\tat java.io.FilterOutputStream.close(FilterOutputStream.java:159)\n\tat com.databricks.backend.daemon.data.server.session.IOStreamManager$OutStream.close(IOStreamManager.scala:155)\n\tat com.databricks.backend.daemon.data.server.handler.IOStreamHandler.receive(IOStreamHandler.scala:25)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.$anonfun$queryHandlers$1(SessionContext.scala:83)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.$anonfun$queryHandlers$1$adapted(SessionContext.scala:82)\n\tat scala.collection.immutable.List.foreach(List.scala:392)\n\tat com.databricks.backend.daemon.data.server.session.SessionContext.queryHandlers(SessionContext.scala:82)\n\tat com.databricks.backend.daemon.data.server.DbfsServerBackend$$anonfun$receive$3.applyOrElse(DbfsServerBackend.scala:272)\n\tat com.databricks.backend.daemon.data.server.DbfsServerBackend$$anonfun$receive$3.applyOrElse(DbfsServerBackend.scala:252)\n\tat com.databricks.rpc.ServerBackend.$anonfun$internalReceive$2(ServerBackend.scala:42)\n\tat com.databricks.rpc.ServerBackend$$anonfun$commonReceive$1.applyOrElse(ServerBackend.scala:58)\n\tat com.databricks.rpc.ServerBackend$$anonfun$commonReceive$1.applyOrElse(ServerBackend.scala:58)\n\tat com.databricks.rpc.ServerBackend.$anonfun$internalReceive$1(ServerBackend.scala:38)\n\tat com.databricks.logging.UsageLogging.$anonfun$recordOperation$2(UsageLogging.scala:359)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.ServerBackend.withAttributionContext(ServerBackend.scala:13)\n\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:268)\n\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:264)\n\tat com.databricks.rpc.ServerBackend.withAttributionTags(ServerBackend.scala:13)\n\tat com.databricks.logging.UsageLogging.recordOperation(UsageLogging.scala:345)\n\tat com.databricks.logging.UsageLogging.recordOperation$(UsageLogging.scala:320)\n\tat com.databricks.rpc.ServerBackend.recordOperation(ServerBackend.scala:13)\n\tat com.databricks.rpc.ServerBackend.internalReceive(ServerBackend.scala:38)\n\tat com.databricks.rpc.JettyServer$RequestManager.$anonfun$handleRPC$2(JettyServer.scala:346)\n\tat scala.util.Try$.apply(Try.scala:213)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleRPC(JettyServer.scala:346)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleRequestAndRespond(JettyServer.scala:269)\n\tat com.databricks.rpc.JettyServer$RequestManager.$anonfun$handleHttp$4(JettyServer.scala:180)\n\tat scala.runtime.java8.JFunction0$mcV$sp.apply(JFunction0$mcV$sp.java:23)\n\tat com.databricks.logging.UsageLogging.$anonfun$withAttributionContext$1(UsageLogging.scala:235)\n\tat scala.util.DynamicVariable.withValue(DynamicVariable.scala:62)\n\tat com.databricks.logging.UsageLogging.withAttributionContext(UsageLogging.scala:230)\n\tat com.databricks.logging.UsageLogging.withAttributionContext$(UsageLogging.scala:227)\n\tat com.databricks.rpc.JettyServer$.withAttributionContext(JettyServer.scala:106)\n\tat com.databricks.logging.UsageLogging.withAttributionTags(UsageLogging.scala:268)\n\tat com.databricks.logging.UsageLogging.withAttributionTags$(UsageLogging.scala:264)\n\tat com.databricks.rpc.JettyServer$.withAttributionTags(JettyServer.scala:106)\n\tat com.databricks.rpc.JettyServer$RequestManager.handleHttp(JettyServer.scala:171)\n\tat com.databricks.rpc.JettyServer$RequestManager.doPost(JettyServer.scala:133)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:707)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n\tat org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:848)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:585)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:515)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:539)\n\tat org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:333)\n\tat org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)\n\tat org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)\n\tat org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:108)\n\tat org.eclipse.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)\n\tat org.eclipse.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)\n\t... 1 more\n</div>"]}}],"execution_count":29},{"cell_type":"code","source":["display(df)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .table-result-container {\n    max-height: 300px;\n    overflow: auto;\n  }\n  table, th, td {\n    border: 1px solid black;\n    border-collapse: collapse;\n  }\n  th, td {\n    padding: 5px;\n  }\n  th {\n    text-align: left;\n  }\n</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>department</th><th>employees</th></tr></thead><tbody><tr><td>List(123456, Computer Science)</td><td>List(List(michael, armbrust, no-reply@berkeley.edu, 100000), List(xiangrui, meng, no-reply@stanford.edu, 120000))</td></tr><tr><td>List(789012, Mechanical Engineering)</td><td>List(List(matei, null, no-reply@waterloo.edu, 140000), List(null, wendell, no-reply@berkeley.edu, 160000), List(michael, armbrust, no-reply@berkeley.edu, 200000))</td></tr><tr><td>List(345678, Theater and Drama)</td><td>List(List(michael, armbrust, no-reply@berkeley.edu, 100000), List(null, wendell, no-reply@berkeley.edu, 160000), List(matei, null, no-reply@waterloo.edu, 180000))</td></tr><tr><td>List(901234, Indoor Recreation)</td><td>List(List(xiangrui, meng, no-reply@stanford.edu, 120000), List(matei, null, no-reply@waterloo.edu, 140000))</td></tr></tbody></table></div>"]}}],"execution_count":30},{"cell_type":"code","source":["from pyspark.sql.functions import explode\ndf = df.select(\"*\",explode(\"employees\").alias(\"e\"))\ndf = df[['department','e']]"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":31},{"cell_type":"code","source":["display(df)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .table-result-container {\n    max-height: 300px;\n    overflow: auto;\n  }\n  table, th, td {\n    border: 1px solid black;\n    border-collapse: collapse;\n  }\n  th, td {\n    padding: 5px;\n  }\n  th {\n    text-align: left;\n  }\n</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>department</th><th>e</th></tr></thead><tbody><tr><td>List(123456, Computer Science)</td><td>List(michael, armbrust, no-reply@berkeley.edu, 100000)</td></tr><tr><td>List(123456, Computer Science)</td><td>List(xiangrui, meng, no-reply@stanford.edu, 120000)</td></tr><tr><td>List(789012, Mechanical Engineering)</td><td>List(matei, null, no-reply@waterloo.edu, 140000)</td></tr><tr><td>List(789012, Mechanical Engineering)</td><td>List(null, wendell, no-reply@berkeley.edu, 160000)</td></tr><tr><td>List(789012, Mechanical Engineering)</td><td>List(michael, armbrust, no-reply@berkeley.edu, 200000)</td></tr><tr><td>List(345678, Theater and Drama)</td><td>List(michael, armbrust, no-reply@berkeley.edu, 100000)</td></tr><tr><td>List(345678, Theater and Drama)</td><td>List(null, wendell, no-reply@berkeley.edu, 160000)</td></tr><tr><td>List(345678, Theater and Drama)</td><td>List(matei, null, no-reply@waterloo.edu, 180000)</td></tr><tr><td>List(901234, Indoor Recreation)</td><td>List(xiangrui, meng, no-reply@stanford.edu, 120000)</td></tr><tr><td>List(901234, Indoor Recreation)</td><td>List(matei, null, no-reply@waterloo.edu, 140000)</td></tr></tbody></table></div>"]}}],"execution_count":32},{"cell_type":"code","source":["df.selectExpr(\"department.*\",\"e.*\").show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+------+--------------------+---------+--------+--------------------+------+\n    id|                name|firstName|lastName|               email|salary|\n+------+--------------------+---------+--------+--------------------+------+\n123456|    Computer Science|  michael|armbrust|no-reply@berkeley...|100000|\n123456|    Computer Science| xiangrui|    meng|no-reply@stanford...|120000|\n789012|Mechanical Engine...|    matei|    null|no-reply@waterloo...|140000|\n789012|Mechanical Engine...|     null| wendell|no-reply@berkeley...|160000|\n789012|Mechanical Engine...|  michael|armbrust|no-reply@berkeley...|200000|\n345678|   Theater and Drama|  michael|armbrust|no-reply@berkeley...|100000|\n345678|   Theater and Drama|     null| wendell|no-reply@berkeley...|160000|\n345678|   Theater and Drama|    matei|    null|no-reply@waterloo...|180000|\n901234|   Indoor Recreation| xiangrui|    meng|no-reply@stanford...|120000|\n901234|   Indoor Recreation|    matei|    null|no-reply@waterloo...|140000|\n+------+--------------------+---------+--------+--------------------+------+\n\n</div>"]}}],"execution_count":33},{"cell_type":"code","source":["from pyspark.sql.types import *\nfrom pyspark.sql.functions import col, count, rand, collect_list, explode, struct, count, lit\nfrom pyspark.sql.functions import pandas_udf, PandasUDFType, udf"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":34},{"cell_type":"code","source":["df = spark.range(0, 10 * 1000 * 1000).withColumn('id', (col('id') / 10000).cast('integer')).withColumn('v', rand())\ndf.cache()\ndf.count()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">108</span><span class=\"ansired\">]: </span>10000000\n</div>"]}}],"execution_count":35},{"cell_type":"code","source":[""],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+---+--------------------+\n id|                   v|\n+---+--------------------+\n  1|  0.4287945836281336|\n  1|0.051267887433541826|\n  1|  0.0768041194975233|\n  1|  0.9167089680640104|\n  1|  0.6803744161180495|\n  1| 0.20734352253930644|\n  1| 0.35501094102739916|\n  1|  0.4767391098562147|\n  1|  0.8783431382029443|\n  1|  0.7341359908160828|\n  1|   0.735751093039033|\n  1|   0.741950929656831|\n  1|  0.1685040989375376|\n  1|  0.1312204610988814|\n  1|  0.8330819961953919|\n  1|  0.6819211434624789|\n  1|  0.4538762994588873|\n  1|0.014623670097911101|\n  1|   0.420909196984551|\n  1|  0.9563449014812565|\n+---+--------------------+\nonly showing top 20 rows\n\n</div>"]}}],"execution_count":36},{"cell_type":"code","source":["from pyspark.sql.types import IntegerType\naddone = udf(lambda s: col(s) + 1, IntegerType())"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":37},{"cell_type":"code","source":["df.withColumn('n',addone(df.v)).count()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">114</span><span class=\"ansired\">]: </span>10000000\n</div>"]}}],"execution_count":38},{"cell_type":"code","source":["@udf('double')\ndef plus_one(v):\n    return v + 1\n\n%timeit df.withColumn('v', plus_one(df.v)).agg(count(col('v'))).show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n1 loops, best of 3: 6.55 s per loop\n</div>"]}}],"execution_count":39},{"cell_type":"code","source":["@pandas_udf(\"double\", PandasUDFType.SCALAR)\ndef pandas_plus_one(v):\n  print (len(v),type(v))\n  return v + 1\n\n#%timeit df.repartition(8).withColumn('v', pandas_plus_one(df.v)).agg(count(col('v'))).show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"]}}],"execution_count":40},{"cell_type":"code","source":["df.rdd.getNumPartitions()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"><span class=\"ansired\">Out[</span><span class=\"ansired\">118</span><span class=\"ansired\">]: </span>8\n</div>"]}}],"execution_count":41},{"cell_type":"code","source":["df.repartition(8).withColumn('v', pandas_plus_one(df.v)).agg(count(col('v'))).show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n</div>"]}}],"execution_count":42},{"cell_type":"code","source":["@pandas_udf(df.schema, PandasUDFType.GROUPED_MAP)\n# Input/output are both a pandas.DataFrame\ndef pandas_subtract_mean(pdf):\n  print (len(pdf),type(pdf))\n  print (pdf.id.unique())\n  return pdf.assign(v=pdf.v - pdf.v.mean())\n\ndf.groupby('id').apply(pandas_subtract_mean).agg(count(col('v'))).show()"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">+--------+\ncount(v)|\n+--------+\n10000000|\n+--------+\n\n</div>"]}}],"execution_count":43},{"cell_type":"code","source":["help(df.groupBy)"],"metadata":{},"outputs":[{"metadata":{},"output_type":"display_data","data":{"text/html":["<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">Help on method groupBy in module pyspark.sql.dataframe:\n\ngroupBy(*cols) method of pyspark.sql.dataframe.DataFrame instance\n    Groups the :class:&#96;DataFrame&#96; using the specified columns,\n    so we can run aggregation on them. See :class:&#96;GroupedData&#96;\n    for all the available aggregate functions.\n    \n    :func:&#96;groupby&#96; is an alias for :func:&#96;groupBy&#96;.\n    \n    :param cols: list of columns to group by.\n        Each element should be a column name (string) or an expression (:class:&#96;Column&#96;).\n    \n    &gt;&gt;&gt; df.groupBy().avg().collect()\n    [Row(avg(age)=3.5)]\n    &gt;&gt;&gt; sorted(df.groupBy(&apos;name&apos;).agg({&apos;age&apos;: &apos;mean&apos;}).collect())\n    [Row(name=&apos;Alice&apos;, avg(age)=2.0), Row(name=&apos;Bob&apos;, avg(age)=5.0)]\n    &gt;&gt;&gt; sorted(df.groupBy(df.name).avg().collect())\n    [Row(name=&apos;Alice&apos;, avg(age)=2.0), Row(name=&apos;Bob&apos;, avg(age)=5.0)]\n    &gt;&gt;&gt; sorted(df.groupBy([&apos;name&apos;, df.age]).count().collect())\n    [Row(name=&apos;Alice&apos;, age=2, count=1), Row(name=&apos;Bob&apos;, age=5, count=1)]\n    \n    .. versionadded:: 1.3\n\n</div>"]}}],"execution_count":44},{"cell_type":"code","source":["\\"],"metadata":{},"outputs":[],"execution_count":45}],"metadata":{"name":"signa2","notebookId":460960499044733},"nbformat":4,"nbformat_minor":0}