多線程復(fù)制文件,同一文件下,多線程并發(fā)寫入同一文件的不同部分。
思路是,提前為每個線程分配好寫入內(nèi)容大小,每個線程執(zhí)行fopen獲取單獨的文件描述符,然后按分配的寫入大小,fseek到不同的位置,并發(fā)寫入內(nèi)容。
但是寫入的內(nèi)容總是錯亂。
#include <stdio.h>
#include <sys/stat.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#define BUFF_SIZE 512
#define PTHREAD_NUMBER 4
typedef struct copy_block
{
char fin[BUFF_SIZE];
char fout[BUFF_SIZE];
long start; //起始位置
long segment_size; //分段大小
int id; //虛擬線程id
} __attribute__((packed)) page;
long file_size(char *filename)
{
struct stat fstat;
memset(&fstat, 0, sizeof(fstat));
stat(filename, &fstat);
return fstat.st_size;
}
//單線程任務(wù)邏輯
void pthread_copy(void *arg)
{
//轉(zhuǎn)換指針類型
page *p = (page *)arg;
//每個線程單獨打開文件
FILE *fin = fopen(p->fin, "r");
FILE *fout = fopen(p->fout, "wb+");
//移動流到偏移位置
int res1 = fseek(fin, p->start, SEEK_SET);
int res2 = fseek(fout, p->start, SEEK_SET);
//開始復(fù)制
char buffer[BUFF_SIZE]; //讀寫區(qū)
long read_size = BUFF_SIZE; //預(yù)設(shè)讀寫大小
long left = p->segment_size; //剩余大小,初始化為任務(wù)總大小
long reade_len = 0; //讀取文件大小
long total_len = 0;
//當(dāng)剩余大小大于0時保持復(fù)制
while (left > 0)
{
//如果文件剩余大小小于預(yù)設(shè)讀寫大小,則按剩余大小讀取
if (read_size > left)
{
read_size = left;
}
//讀取文件
reade_len = fread(buffer, 1, read_size, fin);
total_len += reade_len;
//寫入文件
if (reade_len > 0)
{
fwrite(buffer, 1, reade_len, fout);
}
//剩余大小減去已讀寫大小
left = left - reade_len;
}
//復(fù)制完成關(guān)閉文件
fclose(fin);
fclose(fout);
pthread_exit(NULL);
}
//開啟多線程任務(wù)
int multi_copy(char *src, char *dest)
{
//判斷文件是否存在,以及是否具有讀取權(quán)限
int file_exist = access(src, 4);
if (file_exist != 0)
fprintf(stderr, "源文件不存在");
//獲取文件大小
long fsize = file_size(src);
//真正運行線程數(shù)量
int real_pthread_number = PTHREAD_NUMBER;
if (fsize < PTHREAD_NUMBER)
real_pthread_number = 1;
//給任務(wù)結(jié)構(gòu)體分配內(nèi)存
page *p;
p = malloc(sizeof(*p) * PTHREAD_NUMBER);
long offset = 0; //文本偏移量
long segment = fsize / real_pthread_number; //分段長度
long segment_remainder = fsize % real_pthread_number; //分段后剩余長度
//給每個線程分配任務(wù)
for (int i = 0; i < real_pthread_number; i++)
{
//分配復(fù)制任務(wù)的文件大小
if (i + 1 == real_pthread_number)
{
p[i].segment_size = segment + segment_remainder;
}
else
{
p[i].segment_size = segment;
}
//確定任務(wù)的起止位置
p[i].start = offset;
offset = offset + p[i].segment_size;
//文件路徑存入任務(wù)
strncpy(p[i].fin, src, strlen(src));
strncpy(p[i].fout, dest, strlen(dest));
//分配虛擬線程id
p[i].id = i;
}
//創(chuàng)建線程
pthread_t work[real_pthread_number];
for (int i = 0; i < real_pthread_number; i++)
{
pthread_create(&work[i], NULL, (void *)&pthread_copy, (void *)&p[i]);
}
//阻塞主線程
for (int i = 0; i < real_pthread_number; i++)
{
pthread_join(work[i], NULL);
}
//釋放任務(wù)結(jié)構(gòu)體占用內(nèi)存
if (p != NULL)
{
free(p);
p = NULL;
}
return 0;
}
int main(int argc, char *argv[])
{
char *src;
char *dest;
src = argv[1];
dest = argv[2];
multi_copy(src, dest);
}
一、多線程版本
#include <stdio.h>
#include <sys/stat.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#define BUFF_SIZE 512
#define PTHREAD_NUMBER 4
typedef struct copy_block{
// char src[BUFF_SIZE]; //source file
// char dest[BUFF_SIZE]; //destnation file
FILE *src; //source file
FILE *dest; //destnation file
long start; //begin position
long segment_size; //aim copy size
int id; //virtual thread id
}__attribute__((packed)) Page;
long file_size(const char *filename);
void pthread_copy(void *task_info);
int multi_copy(const char *dest,const char *src);
int main(int argc,char *argv[])
{
if(argc < 3)
{
fprintf(stderr,"Error params!\n");
return 0;
}
char *src = argv[1];
char *dest = argv[2];
multi_copy(dest,src);
return 0;
}
//get file size
long file_size(const char *filename)
{
struct stat fstat;
memset(&fstat,0,sizeof(fstat));
stat(filename,&fstat);
return fstat.st_size;
}
//thread task
void pthread_copy(void *task_info)
{
//convert void * to Page *
Page *p = (Page *)task_info;
//open file for every thread
// FILE *fin = fopen(p->src,"rb");
// FILE *fout = fopen(p->dest,"ab+");
//seek the stream to the offset
int res_in = fseek(p->src,p->start,SEEK_SET);
int res_out = fseek(p->dest,p->start,SEEK_SET);
char buffer[BUFF_SIZE]; //buffer
long read_size = BUFF_SIZE; //the size for read
long left = p->segment_size; //current left bytes
long read_len = 0; //the size has read
long total_len = 0; //the readed total size
printf("Thread %d begin\t",p->id);
printf("start %ld\t",p->start);
printf("segment_size %ld\n",p->segment_size);
// printf("src %s\t",p->src);
// printf("dest %s\n",p->dest);
//begin copy
while(left > 0)
{
if(left < read_size)
{
read_size = left;
}
//read file
read_len = fread(buffer,sizeof(char),read_size,p->src);
total_len += read_len;
//write file
if(read_len > 0)
{
fwrite(buffer,sizeof(char),read_len,p->dest);
}
//the left size
left -= read_len;
}
pthread_exit(NULL);
}
//create multi thread
int multi_copy(const char *dest,const char *src)
{
//file is exists
int file_exists = access(src,04);//read access
if(file_exists != 0)
{
fprintf(stderr,"file is not exists!\n");
return 1;
}
//file size
long filesize = file_size(src);
//pthread number
int real_pthread_number = PTHREAD_NUMBER;
if(filesize < BUFF_SIZE)
{
real_pthread_number = 1;
}
Page *p = (Page *)malloc(sizeof(*p) * real_pthread_number);
long offset = 0; //offset from file begin
long segment = filesize / real_pthread_number; //the bytes of every thread
long segment_remain = filesize % real_pthread_number; //the left bytes for the last thread
//ctor the task_info
//open file for every thread
FILE *fin = fopen(src,"rb");
FILE *fout = fopen(dest,"wb+");
int i;
for(i = 0;i < real_pthread_number;i++)
{
if((i + 1) == real_pthread_number)
{
p[i].segment_size = segment + segment_remain;
}
else
{
p[i].segment_size = segment;
}
//task file pointer offset
p[i].start = offset;
offset += p[i].segment_size;
//file path
// strncpy(p[i].src,src,strlen(src));
// strncpy(p[i].dest,dest,strlen(dest));
p[i].src = fin;
p[i].dest = fout;
//virtual thread id
p[i].id = i;
}
//create thread
pthread_t work[real_pthread_number];
for(i = 0;i < real_pthread_number;i++)
{
pthread_create(&work[i],NULL,(void *)&pthread_copy,(void *)&p[i]);
}
//wait subthread exit
for(i = 0;i < real_pthread_number;i++)
{
pthread_join(work[i],NULL);
}
//free
if(p != NULL)
{
free(p);
p = NULL;
}
fclose(fin);
fclose(fout);
return 0;
}
在main中以rb打開源文件,wb+打開目標(biāo)文件
修改結(jié)構(gòu)體存儲文件指針
在線程pthread_copy中操作文件指針位置,去除打開文件操作(在線程中以wb+打開文件會將文件截斷為0)
在線程調(diào)度函數(shù)multi_copy中打開文件并在末尾關(guān)閉文件
將文件指針賦給每個線程的task_info結(jié)構(gòu)體成員
二、線程鎖版本
#include <stdio.h>
#include <sys/stat.h>
#include <string.h>
#include <pthread.h>
#include <unistd.h>
#include <stdlib.h>
#define BUFF_SIZE 512
#define PTHREAD_NUMBER 4
typedef struct copy_block{
char src[BUFF_SIZE]; //source file
char dest[BUFF_SIZE]; //destnation file
long start; //begin position
long segment_size; //aim copy size
int id; //virtual thread id
}__attribute__((packed)) Page;
static pthread_mutex_t mut;
long file_size(const char *filename);
void pthread_copy(void *task_info);
int multi_copy(const char *dest,const char *src);
int main(int argc,char *argv[])
{
if(argc < 3)
{
fprintf(stderr,"Error params!\n");
return 0;
}
char *src = argv[1];
char *dest = argv[2];
multi_copy(dest,src);
return 0;
}
//get file size
long file_size(const char *filename)
{
struct stat fstat;
memset(&fstat,0,sizeof(fstat));
stat(filename,&fstat);
return fstat.st_size;
}
//thread task
void pthread_copy(void *task_info)
{
pthread_mutex_lock(&mut);
//convert void * to Page *
Page *p = (Page *)task_info;
//open file for every thread
FILE *fin = fopen(p->src,"rb");
FILE *fout = fopen(p->dest,"rb+");
if(fout == NULL)
{
fprintf(stderr,"Open file %s fail\n",p->dest);
return ;
}
//seek the stream to the offset
int res_in = fseek(fin,p->start,SEEK_SET);
int res_out = fseek(fout,p->start,SEEK_SET);
char buffer[BUFF_SIZE]; //buffer
long read_size = BUFF_SIZE; //the size for read
long left = p->segment_size; //current left bytes
long read_len = 0; //the size has read
long total_len = 0; //the readed total size
printf("Thread %d begin\t",p->id);
printf("start %ld\t",p->start);
printf("segment_size %ld\t",p->segment_size);
printf("src %s\t",p->src);
printf("dest %s\n",p->dest);
//begin copy
while(left > 0)
{
if(left < read_size)
{
read_size = left;
}
//read file
read_len = fread(buffer,sizeof(char),read_size,fin);
total_len += read_len;
//write file
if(read_len > 0)
{
fwrite(buffer,sizeof(char),read_len,fout);
}
//the left size
left -= read_len;
}
fclose(fin);
fclose(fout);
pthread_mutex_unlock(&mut);
pthread_exit(NULL);
}
//create multi thread
int multi_copy(const char *dest,const char *src)
{
//file is exists
int file_exists = access(src,04);//read access
if(file_exists != 0)
{
fprintf(stderr,"file is not exists!\n");
return 1;
}
int dest_file_exists = access(dest,04);
if(dest_file_exists != 0)
{
fprintf(stderr,"File %s not exists,create it\n",dest);
FILE *cfp = fopen(dest,"w");
fclose(cfp);
}
//file size
long filesize = file_size(src);
//pthread number
int real_pthread_number = PTHREAD_NUMBER;
if(filesize < BUFF_SIZE)
{
real_pthread_number = 1;
}
Page *p = (Page *)malloc(sizeof(Page) * real_pthread_number);
long offset = 0; //offset from file begin
long segment = filesize / real_pthread_number; //the bytes of every thread
long segment_remain = filesize % real_pthread_number; //the left bytes for the last thread
//ctor the task_info
int i;
for(i = 0;i < real_pthread_number;i++)
{
if((i + 1) == real_pthread_number)
{
p[i].segment_size = segment + segment_remain;
}
else
{
p[i].segment_size = segment;
}
//task file pointer offset
p[i].start = offset;
offset += p[i].segment_size;
//file path
strncpy(p[i].src,src,strlen(src));
strncpy(p[i].dest,dest,strlen(dest));
//virtual thread id
p[i].id = i;
}
//create thread
pthread_mutex_init(&mut,NULL);//init thread lock
pthread_t work[real_pthread_number];
for(i = 0;i < real_pthread_number;i++)
{
pthread_create(&work[i],NULL,(void *)&pthread_copy,(void *)&p[i]);
}
//wait subthread exit
for(i = 0;i < real_pthread_number;i++)
{
pthread_join(work[i],NULL);
}
//free
if(p != NULL)
{
free(p);
p = NULL;
}
return 0;
}
北大青鳥APTECH成立于1999年。依托北京大學(xué)優(yōu)質(zhì)雄厚的教育資源和背景,秉承“教育改變生活”的發(fā)展理念,致力于培養(yǎng)中國IT技能型緊缺人才,是大數(shù)據(jù)專業(yè)的國家
北大青鳥中博軟件學(xué)院創(chuàng)立于2003年,作為華東區(qū)著名互聯(lián)網(wǎng)學(xué)院和江蘇省首批服務(wù)外包人才培訓(xùn)基地,中博成功培育了近30000名軟件工程師走向高薪崗位,合作企業(yè)超4
中公教育集團創(chuàng)建于1999年,經(jīng)過二十年潛心發(fā)展,已由一家北大畢業(yè)生自主創(chuàng)業(yè)的信息技術(shù)與教育服務(wù)機構(gòu),發(fā)展為教育服務(wù)業(yè)的綜合性企業(yè)集團,成為集合面授教學(xué)培訓(xùn)、網(wǎng)
達內(nèi)教育集團成立于2002年,是一家由留學(xué)海歸創(chuàng)辦的高端職業(yè)教育培訓(xùn)機構(gòu),是中國一站式人才培養(yǎng)平臺、一站式人才輸送平臺。2014年4月3日在美國成功上市,融資1
曾工作于聯(lián)想擔(dān)任系統(tǒng)開發(fā)工程師,曾在博彥科技股份有限公司擔(dān)任項目經(jīng)理從事移動互聯(lián)網(wǎng)管理及研發(fā)工作,曾創(chuàng)辦藍懿科技有限責(zé)任公司從事總經(jīng)理職務(wù)負責(zé)iOS教學(xué)及管理工作。
浪潮集團項目經(jīng)理。精通Java與.NET 技術(shù), 熟練的跨平臺面向?qū)ο箝_發(fā)經(jīng)驗,技術(shù)功底深厚。 授課風(fēng)格 授課風(fēng)格清新自然、條理清晰、主次分明、重點難點突出、引人入勝。
精通HTML5和CSS3;Javascript及主流js庫,具有快速界面開發(fā)的能力,對瀏覽器兼容性、前端性能優(yōu)化等有深入理解。精通網(wǎng)頁制作和網(wǎng)頁游戲開發(fā)。
具有10 年的Java 企業(yè)應(yīng)用開發(fā)經(jīng)驗。曾經(jīng)歷任德國Software AG 技術(shù)顧問,美國Dachieve 系統(tǒng)架構(gòu)師,美國AngelEngineers Inc. 系統(tǒng)架構(gòu)師。