2017年9月6日水曜日

[Python][MongoDB]PyMongo:データの挿入

test4.py
#-*- coding: utf-8 -*-
import pymongo
import pytz

#######################################################################
# Main function
#######################################################################
if __name__ == '__main__':
# MongoDBに接続するためのclientを作成する
client = pymongo.MongoClient()

# Databaseに接続する
db = client.test

# Collectionを取得する
collection = db.tbl1

# insert()
data = {
'message': 'thank you',
'value' : 30
}
id = collection.insert(data)
print('id=%s' % (id))

print('\n')
print('find()')
documents = collection.find({'_id':id})
for doc in documents:
#print(doc)
print('%s\t%s\t%d' % (doc['_id'], doc['message'], doc['value']))
実行結果
$ python test4.py
id=59af9cdce138234fdf70d696

find()
59af9cdce138234fdf70d696 thank you 30

[Python][MongoDB]PyMongo

PyMongoはPythonからMongDBを使うためのAPI

インストール

# yum install python-pymongo

データの表示

#-*- coding: utf-8 -*-
import pymongo
import pytz

#######################################################################
# Main function
#######################################################################
if __name__ == '__main__':
# MongoDBに接続するためのclientを作成する
client = pymongo.MongoClient()

# Databaseに接続する
db = client.test

# Collectionを取得する
collection = db.tbl1

# find()
print('find()')
documents = collection.find()
for doc in documents:
#print(doc)
print('%s\t%d' % (doc['message'], doc['value']))

print('\n')
print("find({'message':'hello'})")
documents = collection.find({'message':'hello'})
print('count=%d' % (documents.count()))
for doc in documents:
#print(doc)
print('%s\t%d' % (doc['message'], doc['value']))

print('\n')
print("find({'value':{'$gte':10}})")
documents = collection.find({'value':{'$gte':10}})
print('count=%d' % (documents.count()))
for doc in documents:
#print(doc)
print('%s\t%d' % (doc['message'], doc['value']))
実行結果
$ python test3.py
find()
hello 10
goodbye 25

find({'message':'hello'})
count=1
hello 10

find({'value':{'$gte':10}})
count=2
hello 10
goodbye 25

[Python]jsonモジュール

test0.py
#-*- coding: utf-8 -*-
import json

#######################################################################
# Main function
#######################################################################
if __name__ == '__main__':
# jsonファイル(辞書型)の読み込み
print('=== read test0_dict_read.txt ===')
f = open('test0_dict_read.txt', 'r')
json_dict = json.load(f)
f.close()

print(json_dict)
print(json_dict['book1'])
print(json_dict['book1']["title"])
print(json.dumps(json_dict, indent=4))

# 辞書型をjsonファイルへ書き出し
print('=== write test0_dict_write.txt ===')
json_dict = {}
data = {}
data['name'] = 'tokyo'
data['temperature_high'] = 37.7
data['temperature_low'] = -2.6
json_dict["prefecture_1"] = data

print(json.dumps(json_dict, indent=4))

f = open('test0_dict_write.txt', 'w')
json.dump(json_dict, f)
f.close()

# jsonファイル(配列型)の読み込み
print('=== read test0_list_read.txt ===')
f = open('test0_list_read.txt', 'r')
json_list = json.load(f)
f.close()

print(json_list)
print(json_list[0])
print(json_list[0]['title'])
print(json.dumps(json_list, indent=4))

# 配列型をjsonファイルへ書き出し
print('=== write test0_list_write.txt ===')
json_list = []
data = {}
data['name'] = 'tokyo'
data['temperature_high'] = 37.7
data['temperature_low'] = -2.6
json_list.append(data)

data = {}
data['name'] = 'naha'
data['temperature_high'] = 33.9
data['temperature_low'] = 6.1
json_list.append(data)

print(json.dumps(json_list, indent=4))

f = open('test0_list_write.txt', 'w')
json.dump(json_list, f)
f.close()
test0_dict_read.txt
{
"book1":{
"title": "Python Beginners",
"year": 2005 ,
"page": 399
},
"book2":{
"title": "Python Developers",
"year": 2006 ,
"page": 650
}
}
test0_list_read.txt
[
{
"title": "Python Beginners",
"year": 2005 ,
"page": 399
},
{
"title": "Python Developers",
"year": 2006 ,
"page": 650
}
]
実行結果 コンソール
$ python test0.py
=== read test0_dict_read.txt ===
{u'book1': {u'year': 2005, u'page': 399, u'title': u'Python Beginners'}, u'book2': {u'year': 2006, u'page': 650, u'title': u'Python Developers'}}
{u'year': 2005, u'page': 399, u'title': u'Python Beginners'}
Python Beginners
{
  "book1": {
  "year": 2005,
  "page": 399,
  "title": "Python Beginners"
  },
  "book2": {
  "year": 2006,
  "page": 650,
  "title": "Python Developers"
  }
}
=== write test0_dict_write.txt ===
{
  "prefecture_1": {
  "temperature_high": 37.7,
  "temperature_low": -2.6,
  "name": "tokyo"
  }
}
=== read test0_list_read.txt ===
[{u'year': 2005, u'page': 399, u'title': u'Python Beginners'}, {u'year': 2006, u'page': 650, u'title': u'Python Developers'}]
{u'year': 2005, u'page': 399, u'title': u'Python Beginners'}
Python Beginners
[
  {
  "year": 2005,
  "page": 399,
  "title": "Python Beginners"
  },
  {
  "year": 2006,
  "page": 650,
  "title": "Python Developers"
  }
]
=== write test0_list_write.txt ===
[
  {
  "temperature_high": 37.7,
  "temperature_low": -2.6,
  "name": "tokyo"
  },
  {
  "temperature_high": 33.9,
  "temperature_low": 6.1,
  "name": "naha"
  }
]
test0_dict_write.txt
{"prefecture_1": {"temperature_high": 37.7, "temperature_low": -2.6, "name": "tokyo"}}
test0_list_write.txt
[{"temperature_high": 37.7, "temperature_low": -2.6, "name": "tokyo"}, {"temperature_high": 33.9, "temperature_low": 6.1, "name": "naha"}]

[MongoDB]dump

dump手順

  1. dbpathの場所を確認する。
    # ps aux | grep mongod
    
  2. mongodを止める。
    > use admin
    > db.shutdownServer()
    > quit()
    
  3. dump実行。
    # mongodump --dbpath /var/lib/mongodb --out ~/mongodb_dump_20170906
    

[CentOS][MongoDB]インストール

CentOS 7にMongoDBをインストール・起動する手順

インストール

# yum install mongodb
# yum install mongodb-server

サーバー起動

# systemctl start mongod

自動起動を有効にする

# systemctl enable mongod

[MongoDB]基本操作

データベースの新規作成

> use test
switched to db test

データベースの状態確認

> db.stats()
{
  "db" : "test",
  "collections" : 0,
  "objects" : 0,
  "avgObjSize" : 0,
  "dataSize" : 0,
  "storageSize" : 0,
  "numExtents" : 0,
  "indexes" : 0,
  "indexSize" : 0,
  "fileSize" : 0,
  "dataFileVersion" : {

  },
  "ok" : 1
}

データベースの削除

> use test
> db.dropDatabase()

データ挿入

> db.tbl1.insert({message:'hello', value:10})
WriteResult({ "nInserted" : 1 })
> db.tbl1.insert({message:'goodbye', value:20})
WriteResult({ "nInserted" : 1 })

登録データの表示(SELECT * FROM tbl1;)

> db.tbl1.find()
{ "_id" : ObjectId("59af8c93ce7c2c41694720f5"), "message" : "hello", "value" : 10 }
{ "_id" : ObjectId("59af8ca5ce7c2c41694720f6"), "message" : "goodbye", "value" : 20 }

条件指定しての検索(SELECT)

> db.tbl1.find({"message":"hello"})
{ "_id" : ObjectId("59af8c93ce7c2c41694720f5"), "message" : "hello", "value" : 10 }
> db.tbl1.find({"value":{$gt:15}})
{ "_id" : ObjectId("59af8ca5ce7c2c41694720f6"), "message" : "goodbye", "value" : 20 }
> db.tbl1.find({"value":{$gte:10}})
{ "_id" : ObjectId("59af8c93ce7c2c41694720f5"), "message" : "hello", "value" : 10 }
{ "_id" : ObjectId("59af8ca5ce7c2c41694720f6"), "message" : "goodbye", "value" : 20 }

データの更新(UPDATE)

> db.tbl1.find({"message":"goodbye"})
{ "_id" : ObjectId("59af8ca5ce7c2c41694720f6"), "message" : "goodbye", "value" : 20 }
> db.tbl1.update({"message":"goodbye"},{$set:{"value":25}})
WriteResult({ "nMatched" : 1, "nUpserted" : 0, "nModified" : 1 })
> db.tbl1.find({"message":"goodbye"})
{ "_id" : ObjectId("59af8ca5ce7c2c41694720f6"), "message" : "goodbye", "value" : 25 }

DB一覧表示

> show dbs;

コレクション一覧表示

> use test
switched to db test
> show collections;

2017年6月28日水曜日

[D3.js]csvファイルをテーブルで表示する

test1.html
<html>
  <head>
    <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
    <script src="https://d3js.org/d3.v4.js"></script>
    <link rel="stylesheet" type="text/css" href="./test1.css">
  </head>
  <body>
  <svg width="960" height="500"></svg><br/>
  <a href="test1.csv">下表をcsvファイルにしてダウンロード</a><br/>
  <table></table>
  </body>
  <!-- 上記のsvg要素をJavaScript内で読み込むので、bodyより下にスクリプトのソースを定義する -->
  <script src="./test1.js"></script>
</html>
test1.js
var svg = d3.select("svg"),
    margin = {top: 20, right: 80, bottom: 30, left: 50},
    width = svg.attr("width") - margin.left - margin.right,
    height = svg.attr("height") - margin.top - margin.bottom,
    g = svg.append("g").attr("transform", "translate(" + margin.left + "," + margin.top + ")");

var line = d3.line()
             .x(function(d) { return x(d.month); })
             .y(function(d) { return y(d.temperature); });

// X, Yの範囲を座標に置き換える
var x = d3.scaleLinear()
    .range([0, width]);

var y = d3.scaleLinear()
    .range([height, 0]);

// 色
var colors = d3.scaleOrdinal(d3.schemeCategory10);
console.log(colors(0))
console.log(colors(1))
console.log(colors(2))
console.log(colors(3))

// データファイルを読み込んでグラフを描く
d3.csv("test1.csv", type, function(error, data) {
 if (error) throw error;

 var cities = data.columns.slice(1).map(function(id) {
  return {
   id: id,
   values: data.map(function(d) {
    return {month: d.month, temperature: d[id]};
   })
  };
 });

 var keys = data.columns.slice(1);
 console.log('keys=' + keys);

 // Xの範囲はmonthの最小値から最大値
 console.log('extent=' + d3.extent(data, function(d) { return d.month; }));
 console.log('max=' + d3.max(data, function(d) { return d.month; }));
 x.domain(d3.extent(data, function(d) {
  return d.month
 }));
 console.log('x.domain()=' + x.domain())
 console.log('x.range()=' + x.range())

 // Yの範囲は各地のtemperatureをすべて含めた最小値から最大値
 y_min = d3.min(cities, function(c) { 
  return d3.min(c.values, function(d) {
   return d.temperature;
  });
 });
 y_max = d3.max(cities, function(c) { return d3.max(c.values, function(d) { return d.temperature; }); });
 console.log('y_min=' + y_min);
 console.log('y_max=' + y_max);
 y.domain([y_min, y_max]);

 // X軸 (y=0の場所配置するためにtranslateでy(0)を指定する
 g.append("g")
  .attr("transform", "translate(0," + y(0) + ")")
  .call(d3.axisBottom(x));

 // Y軸
 g.append("g")
  .call(d3.axisLeft(y))
  .append("text")
    .attr("transform", "rotate(-90)")
    .attr("y", 6)
    .attr("dy", "0.71em")
    .attr("fill", "#000")
    .text("Temperature (℃)");

 // line chartを描く
 var city = g.selectAll(".city")
             .data(cities)
             .enter().append("g")
               .attr("class", "city");

 city.append("path")
  .attr("d", function(d) {
    return line(d.values);
  })
  .attr("stroke", function(d, i) {
   console.log('d.id=' + d.id);
   console.log('i=' + i);
   console.log('colors(d.id)=' + colors(d.id));
   console.log('colors(i)=' + colors(i));
     return colors(i);
  })
  .attr("stroke-width", 2)
  .attr("fill", "none");

 // 凡例
 console.log('keys.slice()=' + keys.slice());
 var legend = g.append("g")
               .attr("font-family", "sans-serif")
               .attr("font-size", 10)
               .attr("text-anchor", "end")
               .selectAll("g")
               //.data(['tokyo', 'osaka', 'sapporo', 'naha'])
               .data(keys.slice())
               .enter()
               .append("g")
               .attr("transform", function(d, i) {
                  console.log(i)
                     return "translate(0," + i * 20 + ")"; });
 legend.append("path")
       .attr("d", function(d) { return "M" + (width-19) + ",10" + "L" + (width) + ",10"; })
       .attr("stroke", function(d, i) {
          console.log('colors(' + i + ')=' + colors(i));
          return colors(i);
       })
       .attr("stroke-width", 2)
       .attr("fill", "none");

 legend.append("text")
       .attr("x", width - 24)
       .attr("y", 9.5)
       .attr("dy", "0.32em")
       .text(function(d) { return d; });
});

var table = d3.select("table");
var thead = table.append("thead");
var tbody = table.append("tbody");

d3.csv("test1.csv", function(csv) {
 var headerKey = d3.map(csv[0]).keys();

 thead.append("tr")
      .selectAll("th")
      .data(headerKey)
      .enter()
      .append("th")
      .text(function(key) { return key; });

 tbody.selectAll("tr")
      .data(csv)
      .enter()
      .append("tr")
        .selectAll("td")
        .data(function(row) { return d3.entries(row); })
        .enter()
        .append("td")
        .text(function(d) { return d.value; })
})

function type(d, _, columns) {
 d.month = +d.month; // 文字列を数値にする
 for (var i = 1, n = columns.length, c; i < n; ++i) {
  d[c = columns[i]] = +d[c];
 }

 return d;
}
test1.css
table {
 border-collapse: collapse;
 text-align: left;
 line-height: 1.5;
}

th {
 padding: 10px;
 font-weight: bold;
 vertical-align: top;
 color: #369;
 border-bottom: 3px solid #036;
}

td {
 weight: 350px;
 padding: 10px;
 vertical-align: top;
 border-bottom: 1px solid #ccc;
}
test1.csv
month,tokyo,osaka,sapporo,naha
1,5.1,4.4,-3.8,14.9
2,7,7.4,-1.1,17.6
3,8.1,8.1,0.7,17.1
4,14.5,13.8,6.9,20.4
5,18.5,19.6,11.1,23.9
6,22.8,24.2,17.3,27.9
7,27.3,27.8,21.8,28.9
8,27.5,28.9,26.6,28.3
9,25.1,25.2,19.2,27.9
10,19.5,19.5,12.1,25.2
11,14.9,15.2,6,23.7
12,7.5,8.1,-2,18.6
実行例 (fc2に置いてあるページに飛びます)