Out of the Box

사용자 도구

사이트 도구

추적: • httpx • 2020-08_chess_transformer_mastering_play_using_generative_language_models • anaconda • speedup • asciimatics • pywebio
code:gpt_example

차이

문서의 선택한 두 판 사이의 차이를 보여줍니다.

차이 보기로 링크

양쪽 이전 판이전 판
다음 판		이전 판
code:gpt_example [2020/08/21 15:23] – rex8312code:gpt_example [2024/03/23 02:42] (현재) – 바깥 편집 127.0.0.1
줄 1: 줄 1:
-====== [example] GPT ======+====== Example: GPT ======
  
   * 참고   * 참고
줄 41: 줄 41:
     parser.add_argument('--noise_scale', type=float, default=0.1)     parser.add_argument('--noise_scale', type=float, default=0.1)
     parser.add_argument('--max_grad_norm', type=float, default=1.0)     parser.add_argument('--max_grad_norm', type=float, default=1.0)
 +    parser.add_argument('--dataset', choices=['BasicDataset', 'MotionDataset'], default='MotionDataset')
     return parser.parse_args()     return parser.parse_args()
  
줄 119: 줄 120:
         self.block_size = block_size         self.block_size = block_size
  
-        self.we = nn.Linear(input_dims, self.d_model, bias=False)+        self.we = nn.Linear(input_dims, self.d_model, bias=True)
         self.wp = nn.Parameter(torch.zeros(1, self.block_size, self.d_model))         self.wp = nn.Parameter(torch.zeros(1, self.block_size, self.d_model))
         self.blocks = nn.Sequential(*[         self.blocks = nn.Sequential(*[
줄 126: 줄 127:
         ])         ])
         self.norm = nn.LayerNorm(self.d_model)         self.norm = nn.LayerNorm(self.d_model)
-        self.wd = nn.Linear(self.d_model, output_dims, bias=False)+        self.wd = nn.Linear(self.d_model, output_dims, bias=True)
  
         self.apply(self._init_weights)         self.apply(self._init_weights)
줄 163: 줄 164:
         self.data = self.data.astype(np.float32)         self.data = self.data.astype(np.float32)
         self.data = self.data.reshape(-1, 1)         self.data = self.data.reshape(-1, 1)
 +        self.data_std = self.data.std(0)
         self.repeat = repeat         self.repeat = repeat
         self.noise_scale = noise_scale         self.noise_scale = noise_scale
줄 174: 줄 176:
         i = np.random.randint(0, len(self.data) - (self.block_size + 1))         i = np.random.randint(0, len(self.data) - (self.block_size + 1))
         chunk = self.data[i: i+self.block_size+1]         chunk = self.data[i: i+self.block_size+1]
-        chunk += np.random.normal(0, args.noise_scale, chunk.shape)+        chunk += np.random.normal(0, args.noise_scale, chunk.shape) * self.data_std
         x = torch.tensor(chunk[:-1], dtype=torch.float32)         x = torch.tensor(chunk[:-1], dtype=torch.float32)
         y = torch.tensor(chunk[1:], dtype=torch.float32)         y = torch.tensor(chunk[1:], dtype=torch.float32)
줄 183: 줄 185:
         idx = np.arange(i, i+test_steps)         idx = np.arange(i, i+test_steps)
         data = self.data[idx].reshape(1, -1, 1)         data = self.data[idx].reshape(1, -1, 1)
 +        tgt = torch.tensor(data, device=device)
 +        src = tgt[:, :args.block_size]
 +        gen = tgt[:, :args.block_size]
 +        return tgt, src, gen
 +
 +
 +class MotionDataset(Dataset):
 +    
 +    def __init__(self, block_size, repeat, noise_scale):
 +        self.block_size = block_size
 +
 +        import urllib, json
 +        url = "https://raw.githubusercontent.com/xbpeng/DeepMimic/master/data/motions/humanoid3d_backflip.txt"
 +        self.data = json.loads(urllib.request.urlopen(url).read())['Frames']
 +        self.data = np.array(self.data, dtype=np.float32)
 +        self.data = np.hstack([self.data[:, 3:4], self.data])
 +        self.data = np.tile(self.data, (100, 1))
 +        self.dims = self.data.shape[-1]
 +        self.data_mean = self.data.mean(0, keepdims=True)
 +        self.data_std = self.data.std(0, keepdims=True)
 +        self.data = (self.data - self.data_mean) / self.data_std
 +
 +        self.data = self.data.astype(np.float32)
 +        self.repeat = repeat
 +        self.noise_scale = noise_scale
 +    
 +    def __len__(self):
 +        # return math.ceil(len(self.data) / (self.block_size + 1))
 +        return len(self.data) * self.repeat
 +
 +    def __getitem__(self, idx):
 +        # we're actually going to "cheat" and pick a spot in the dataset at random
 +        i = np.random.randint(0, len(self.data) - (self.block_size + 1))
 +        chunk = self.data[i: i+self.block_size+1]
 +        chunk += np.random.normal(0, args.noise_scale, chunk.shape)
 +        x = torch.tensor(chunk[:-1], dtype=torch.float32)
 +        y = torch.tensor(chunk[1:], dtype=torch.float32)
 +        return x, y
 +
 +    def get_test_data(self, test_steps, device):
 +        i = np.random.randint(0, len(self.data) - (test_steps + 1))
 +        idx = np.arange(i, i+test_steps)
 +        data = self.data[idx].reshape(1, -1, self.dims)
         tgt = torch.tensor(data, device=device)         tgt = torch.tensor(data, device=device)
         src = tgt[:, :args.block_size]         src = tgt[:, :args.block_size]
줄 192: 줄 237:
  
     # create the dataloader     # create the dataloader
-    dataset = BasicDataset(args.block_size, args.data_repeat, args.noise_scale)+    Dataset = globals()[args.dataset] 
 +    dataset = Dataset(args.block_size, args.data_repeat, args.noise_scale)
     loader = DataLoader(dataset, batch_size=args.batch_size, num_workers=args.n_workers)     loader = DataLoader(dataset, batch_size=args.batch_size, num_workers=args.n_workers)
          
     # create the model     # create the model
-    model = GPTModel(1, 1, args.block_size).to(args.device)+    dim = dataset.data.shape[-1] 
 +    model = GPTModel(dim, dim, args.block_size).to(args.device)
  
     # create the optimizer     # create the optimizer
줄 216: 줄 263:
     step = 0     step = 0
     train_loss_list = list()     train_loss_list = list()
-    test_loss_list = list()+    test_score_list = list()
          
     for epoch in tqdm.trange(args.max_epoch):     for epoch in tqdm.trange(args.max_epoch):
줄 248: 줄 295:
                  
         loss = (0.5 * (tgt - gen) ** 2).mean()         loss = (0.5 * (tgt - gen) ** 2).mean()
-        test_loss_list.append((step, loss.item()))+        score = (-loss).exp() 
 +        test_score_list.append((step, score.item()))
  
-        mlab.plot(tgt.squeeze_().cpu().numpy()) +        mlab.plot(tgt.cpu().numpy()[0, :, 0]) 
-        mlab.oplot(gen.squeeze_().cpu().numpy()) +        mlab.oplot(gen.cpu().numpy()[0, :, 0]) 
-        # tqdm.tqdm.write(plotille.scatter(*zip(*test_loss_list[-1000:]), height=25))+        tqdm.tqdm.write(plotille.scatter(*zip(*test_score_list[-1000:]), height=25))
         tqdm.tqdm.write(str(args))         tqdm.tqdm.write(str(args))
  
     embed()     embed()
 +
 </code> </code>
  
code/gpt_example.1598023422.txt.gz · 마지막으로 수정됨: (바깥 편집)