Is the newly released CodeLlama 70B amazing, not only byHumanEval numbers but also in real world tasks?
The test prompt:
Modify the following code，define an interface DataHandler for the structure of DataFileHandler. create a new class DataSQLHandler to implement the DataHandler interface and use sqlite to implement similar functions of DataFileHandler.
import pandas as pd
import os
import signal
from datetime import datetime
from email.utils import formatdate
import sys
class DataFileHandler:
def __init__(self, main_key, site, query):
self.main_key = main_key
self.site = site
self.run_start_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.main_csv_file = f'{self.site}-{query}.csv'
self.temp_csv_file = f'{self.site}-{self.run_start_time}.csv'
self.last_saved_main_key = self.get_last_saved_key()
print_err(f"last_saved_main_key={self.last_saved_main_key}")
signal.signal(signal.SIGINT, self.save_main_file)
def save_main_file(self, signum=None, frame=None):#signal handler(signum=None, frame=None)
if os.path.exists(self.temp_csv_file):
temp_df = pd.read_csv(self.temp_csv_file)
if not temp_df.empty:
reversed_df = temp_df.iloc[::-1]
if os.path.exists(self.main_csv_file):
reversed_df.to_csv(self.main_csv_file, mode='a', index=False, header=False)
else:
reversed_df.to_csv(self.main_csv_file, mode='w', index=False, header=True)
os.remove(self.temp_csv_file)
def append_data(self, data):
data['record_time'] = formatdate(localtime=True, usegmt=True)
df = pd.DataFrame([data])
if not os.path.exists(self.temp_csv_file):
df.to_csv(self.temp_csv_file, mode='w', index=False, header=True)
else:
df.to_csv(self.temp_csv_file, mode='a', index=False, header=False)
def get_last_saved_key(self):
if os.path.exists(self.main_csv_file):
df = pd.read_csv(self.main_csv_file)
if not df.empty:
return df.iloc[-1][self.main_key]
return None
def is_duplicate(self, current_data_key):
return current_data_key == self.last_saved_main_keyGPT-4 was prompted in Chinese (same meaning, slightly different wording) and the code is almost perfect. i don’t bother to include its explanation text here.
import sqlite3
from abc import ABC, abstractmethod
class DataHandler(ABC):
@abstractmethod
def save_main_file(self, signum=None, frame=None):
pass
@abstractmethod
def append_data(self, data):
pass
@abstractmethod
def get_last_saved_key(self):
pass
@abstractmethod
def is_duplicate(self, current_data_key):
pass
# DataFileHandler 现在继承自 DataHandler，并实现其方法
class DataFileHandler(DataHandler):
# [原有的 DataFileHandler 类的实现]
...
class DataSQLHandler(DataHandler):
def __init__(self, main_key, site, query, db_file='data.db'):
self.main_key = main_key
self.site = site
self.db_file = db_file
self.table_name = f'{site}_{query}'
self.create_table()
self.last_saved_main_key = self.get_last_saved_key()
def create_table(self):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f'''CREATE TABLE IF NOT EXISTS {self.table_name}
(record_time TEXT, {self.main_key} TEXT, PRIMARY KEY({self.main_key}))''')
conn.commit()
conn.close()
def save_main_file(self, signum=None, frame=None):
# 在SQLite中，此方法可能不需要，因为数据已经在数据库中(this means: In SQLite this method may not be needed as the data is already in the database)
pass
def append_data(self, data):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
columns = ', '.join(data.keys())
placeholders = ':'+', :'.join(data.keys())
c.execute(f'INSERT OR IGNORE INTO {self.table_name} ({columns}) VALUES ({placeholders})', data)
conn.commit()
conn.close()
def get_last_saved_key(self):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f'SELECT {self.main_key} FROM {self.table_name} ORDER BY ROWID DESC LIMIT 1')
result = c.fetchone()
conn.close()
return result[0] if result else None
def is_duplicate(self, current_data_key):
return current_data_key == self.last_saved_main_keyMistral-medium (hosted by perplexity) is at the same level of GPT-4!
Here’s the modified code with the requested changes:
import sqlite3
import pandas as pd
import os
import signal
from datetime import datetime
from email.utils import formatdate
import sys
class DataHandler:
def append_data(self, data):
pass
def get_last_saved_key(self):
pass
def is_duplicate(self, current_data_key):
pass
class DataFileHandler(DataHandler):
#identical copy of the codein prompt
#...
class DataSQLHandler(DataHandler):
def __init__(self, main_key, site, query):
self.main_key = main_key
self.site = site
self.run_start_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.conn = sqlite3.connect(f'{self.site}-{query}.db')
self.cursor = self.conn.cursor()
self.cursor.execute('''CREATE TABLE IF NOT EXISTS data
(record_time TEXT, {main_key} TEXT)'''.format(main_key=self.main_key))
self.last_saved_main_key = self.get_last_saved_key()
print_err(f"last_saved_main_key={self.last_saved_main_key}")
signal.signal(signal.SIGINT, self.save_main_file)
def save_main_file(self, signum=None, frame=None):
pass
def append_data(self, data):
data['record_time'] = formatdate(localtime=True, usegmt=True)
self.cursor.execute('''INSERT INTO data VALUES (?, ?)''',
(data['record_time'], data[self.main_key]))
self.conn.commit()
def get_last_saved_key(self):
self.cursor.execute('''SELECT {main_key} FROM data ORDER BY record_time DESC LIMIT 1'''.format(main_key=self.main_key))
result = self.cursor.fetchone()
if result:
return result[0]
return None
def is_duplicate(self, current_data_key):
self.cursor.execute('''SELECT COUNT(*) FROM data WHERE {main_key}=? ORDER BY record_time DESC LIMIT 1'''.format(main_key=self.main_key),
(current_data_key,))
count = self.cursor.fetchone()[0]
return count > 0In the above code, I have defined an interface `DataHandler` with three abstract methods: `append_data`, `get_last_saved_key`, and `is_duplicate`. Then, I have modified the original `DataFileHandler` class to implement this interface and added a new class `DataSQLiteHandler` that also implements the `DataHandler` interface and uses SQLite to store data instead of CSV files.
Note that the `save_main_file` method is not needed in the `DataSQLiteHandler` class since SQLite automatically handles data persistence. Also, the `is_duplicate` method in the `DataSQLiteHandler` class uses a SQL query to check if the current data key already exists in the database.
I’m disappointed by CodeLlama-70B-instruct.
1. Define an interface DataHandler for the structure of DataFileHandler.
class DataHandler:
def __init__(self, main_key, site, query):
pass
def save_main_file(self):
pass
def append_data(self, data):
pass
def get_last_saved_key(self):
pass
def is_duplicate(self, current_data_key):
pass2. Create a new class DataSQLHandler to implement the DataHandler interface and use sqlite to implement similar functions of DataFileHandler.
import sqlite3
class DataSQLHandler(DataHandler):
def __init__(self, main_key, site, query):
self.main_key = main_key
self.site = site
self.run_start_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.main_table = f'{self.site}-{query}'
self.temp_table = f'{self.site}-{self.run_start_time}'
self.last_saved_main_key = self.get_last_saved_key()
print_err(f"last_saved_main_key={self.last_saved_main_key}")
signal.signal(signal.SIGINT, self.save_main_file)
def save_main_file(self, signum=None, frame=None):#signal handler(signum=None, frame=None)
if self.temp_table_exists():
temp_df = self.get_temp_table_data()
if not temp_df.empty:
reversed_df = temp_df.iloc[::-1]
if self.main_table_exists():
self.append_to_main_table(reversed_df)
else:
self.create_main_table(reversed_df)
self.drop_temp_table()
def append_data(self, data):
data['record_time'] = formatdate(localtime=True, usegmt=True)
df = pd.DataFrame([data])
if not self.temp_table_exists():
self.create_temp_table(df)
else:
self.append_to_temp_table(df)
def get_last_saved_key(self):
if self.main_table_exists():
df = self.get_main_table_data()
if not df.empty:
return df.iloc[-1][self.main_key]
return None
def is_duplicate(self, current_data_key):
return current_data_key == self.last_saved_main_key
def temp_table_exists(self):
pass
def main_table_exists(self):
pass
def get_temp_table_data(self):
pass
def get_main_table_data(self):
pass
def append_to_temp_table(self, df):
pass
def append_to_main_table(self, df):
pass
def create_temp_table(self, df):
pass
def create_main_table(self, df):
pass
def drop_temp_table(self):
passIn the DataSQLHandler class, the methods `temp_table_exists`, `main_table_exists`, `get_temp_table_data`, `get_main_table_data`, `append_to_temp_table`, `append_to_main_table`, `create_temp_table`, `create_main_table`, and `drop_temp_table` are left blank as they require specific implementation details for sqlite.
Try again.
1. Create a new class DataSQLHandler to implement the DataHandler interface and use sqlite to implement similar functions of DataFileHandler.
import sqlite3
import os
import signal
from datetime import datetime
from email.utils import formatdate
import sys
class DataSQLHandler:
def __init__(self, main_key, site, query):
self.main_key = main_key
self.site = site
self.run_start_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.db_file = f'{self.site}-{query}.db'
self.table_name = f'{self.site}-{self.run_start_time}'
self.last_saved_main_key = self.get_last_saved_key()
print_err(f"last_saved_main_key={self.last_saved_main_key}")
signal.signal(signal.SIGINT, self.save_main_file)
def save_main_file(self, signum=None, frame=None):#signal handler(signum=None, frame=None)
if os.path.exists(self.db_file):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f"SELECT * FROM {self.table_name}")
rows = c.fetchall()
if rows:
reversed_rows = rows[::-1]
if os.path.exists(self.main_csv_file):
with open(self.main_csv_file, 'a') as f:
for row in reversed_rows:
f.write(','.join(row) + '\n')
else:
with open(self.main_csv_file, 'w') as f:
for row in reversed_rows:
f.write(','.join(row) + '\n')
conn.close()
def append_data(self, data):
data['record_time'] = formatdate(localtime=True, usegmt=True)
if not os.path.exists(self.db_file):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f"CREATE TABLE {self.table_name} (main_key text, record_time text)")
conn.commit()
conn.close()
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f"INSERT INTO {self.table_name} VALUES (?, ?)", (data['main_key'], data['record_time']))
conn.commit()
conn.close()
def get_last_saved_key(self):
if os.path.exists(self.db_file):
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
c.execute(f"SELECT * FROM {self.table_name}")
rows = c.fetchall()
if rows:
return rows[-1][0]
return NoneNew class looks better but it forgot the 1st task to define an interface.
So I tried his little brother CodeLlama-34B-instruct also.
To create an interface `DataHandler` for the structure of `DataFileHandler`, you can define the interface as follows:
from abc import ABC, abstractmethod
class DataHandler(ABC):
@abstractmethod
def save_main_file(self, signum=None, frame=None):
pass
@abstractmethod
def append_data(self, data):
pass
@abstractmethod
def get_last_saved_key(self):
pass
@abstractmethod
def is_duplicate(self, current_data_key):
passThis interface defines four methods that are common to both `DataFileHandler` and `DataSQLHandler`: `save_main_file`, `append_data`, `get_last_saved_key`, and `is_duplicate`. These methods are marked as abstract, meaning that they must be implemented by any class that implements the `DataHandler` interface.
To create a new class `DataSQLHandler` that implements the `DataHandler` interface and uses SQLite to implement similar functions of `DataFileHandler`, you can define the class as follows:
import sqlite3
class DataSQLHandler(DataHandler):
def __init__(self, main_key, site, query):
self.main_key = main_key
self.site = site
self.run_start_time = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
self.main_db_file = f'{self.site}-{query}.db'
self.temp_db_file = f'{self.site}-{self.run_start_time}.db'
self.last_saved_main_key = self.get_last_saved_key()
print_err(f"last_saved_main_key={self.last_saved_main_key}")
signal.signal(signal.SIGINT, self.save_main_file)
def save_main_file(self, signum=None, frame=None):
if os.path.exists(self.temp_db_file):
temp_db = sqlite3.connect(self.temp_db_file)
temp_db.commit()
temp_db.close()
os.remove(self.temp_db_file)
def append_data(self, data):
data['record_time'] = formatdate(localtime=True, usegmt=True)
db = sqlite3.connect(self.main_db_file)
db.execute("INSERT INTO data (record_time, data) VALUES (?, ?)", (data['record_time'], data))
db.commit()
db.close()
def get_last_saved_key(self):
db = sqlite3.connect(self.main_db_file)
cursor = db.execute("SELECT * FROM data ORDER BY record_time DESC LIMIT 1")
last_row = cursor.fetchone()
db.close()
if last_row:
return last_row[0]
return None
def is_duplicate(self, current_data_key):
db = sqlite3.connect(self.main_db_file)
cursor = db.execute("SELECT * FROM data WHERE record_time = ?", (current_data_key,))
result = cursor.fetchone()
db.close()
return result is not NoneThis class defines the same methods as `DataFileHandler`, but uses SQLite to store the data instead of a CSV file. The `save_main_file` method commits the changes to the temporary database file and then deletes the file. The `append_data` method inserts the data into the main database file. The `get_last_saved_key` method retrieves the last saved key from the main database file. The `is_duplicate` method checks if the current data key is already in the main database file.
To use the `DataSQLHandler` class, you can create an instance of it and call its methods in the same way as you would with `DataFileHandler`:
data_handler = DataSQLHandler('main_key', 'site', 'query')
data_handler.append_data({'key': 'value'})
data_handler.save_main_file()This will create a new SQLite database file with the data appended to it, and then commit the changes to the file.
This seems even better than the 70B!
One more test is here.
I’m much more curious about how large is mistral-medium now.
Another purpose of conducting these experiments is to study the assistive role of AI code tools and LLM services in programming and learning new technologies. This will be discussed in another series of articles, and feedback is welcome.
