File: /usr/src/linux/arch/parisc/kernel/sba_iommu.c
1 /*
2 ** System Bus Adapter (SBA) I/O MMU manager
3 **
4 ** (c) Copyright 2000 Grant Grundler
5 ** (c) Copyright 2000 Hewlett-Packard Company
6 **
7 ** Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
8 **
9 ** This program is free software; you can redistribute it and/or modify
10 ** it under the terms of the GNU General Public License as published by
11 ** the Free Software Foundation; either version 2 of the License, or
12 ** (at your option) any later version.
13 **
14 **
15 ** This module initializes the IOC (I/O Controller) found on B1000/C3000/
16 ** J5000/J7000/N-class/L-class machines and their successors.
17 **
18 ** FIXME: Multi-IOC support missing - depends on hp_device data
19 ** FIXME: add DMA hint support programming in both sba and lba modules.
20 */
21
22 #include <linux/config.h>
23 #include <linux/types.h>
24 #include <linux/kernel.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/init.h>
28
29 #include <linux/mm.h>
30 #include <linux/string.h>
31 #define PCI_DEBUG /* for ASSERT */
32 #include <linux/pci.h>
33 #undef PCI_DEBUG
34
35 #include <asm/byteorder.h>
36 #include <asm/io.h>
37 #include <asm/dma.h> /* for DMA_CHUNK_SIZE */
38
39 #include <asm/hardware.h> /* for register_driver() stuff */
40 #include <asm/gsc.h> /* FIXME: for gsc_read/gsc_write */
41
42 #include <linux/proc_fs.h>
43 #include <asm/runway.h> /* for proc_runway_root */
44
45
46 #define MODULE_NAME "SBA"
47
48 /*
49 ** The number of debug flags is a clue - this code is fragile.
50 ** Don't even think about messing with it unless you have
51 ** plenty of 710's to sacrafice to the computer gods. :^)
52 */
53 #undef DEBUG_SBA_INIT
54 #undef DEBUG_SBA_RUN
55 #undef DEBUG_SBA_RUN_SG
56 #undef DEBUG_SBA_RESOURCE
57 #undef ASSERT_PDIR_SANITY
58 #undef DEBUG_LARGE_SG_ENTRIES
59
60 #if 1
61 #define SBA_INLINE
62 #else
63 #define SBA_INLINE __inline__
64 #endif
65
66 #ifdef DEBUG_SBA_INIT
67 #define DBG_INIT(x...) printk(x)
68 #else
69 #define DBG_INIT(x...)
70 #endif
71
72 #ifdef DEBUG_SBA_RUN
73 #define DBG_RUN(x...) printk(x)
74 #else
75 #define DBG_RUN(x...)
76 #endif
77
78 #ifdef DEBUG_SBA_RUN_SG
79 #define DBG_RUN_SG(x...) printk(x)
80 #else
81 #define DBG_RUN_SG(x...)
82 #endif
83
84
85 #ifdef DEBUG_SBA_RESOURCE
86 #define DBG_RES(x...) printk(x)
87 #else
88 #define DBG_RES(x...)
89 #endif
90
91 /*
92 ** The number of pdir entries to "free" before issueing
93 ** a read to PCOM register to flush out PCOM writes.
94 ** Interacts with allocation granularity (ie 4 or 8 entries
95 ** allocated and free'd/purged at a time might make this
96 ** less interesting).
97 */
98 #if 0
99 #define DELAYED_RESOURCE_CNT 16
100 #else
101 #undef DELAYED_RESOURCE_CNT
102 #endif
103
104 #define DEFAULT_DMA_HINT_REG 0
105
106 #define ASTRO_RUNWAY_PORT 0x582
107 #define ASTRO_ROPES_PORT 0x780
108
109 #define IKE_MERCED_PORT 0x803
110 #define IKE_ROPES_PORT 0x781
111
112 int sba_driver_callback(struct hp_device *, struct pa_iodc_driver *);
113
114 static struct pa_iodc_driver sba_drivers_for[] = {
115
116 /* FIXME: why is SVERSION checked? */
117
118 {HPHW_IOA, ASTRO_RUNWAY_PORT, 0x0, 0xb, 0, 0x10,
119 DRIVER_CHECK_HVERSION +
120 DRIVER_CHECK_SVERSION + DRIVER_CHECK_HWTYPE,
121 MODULE_NAME, "I/O MMU", (void *) sba_driver_callback},
122
123 {HPHW_BCPORT, ASTRO_ROPES_PORT, 0x0, 0xb, 0, 0x10,
124 DRIVER_CHECK_HVERSION +
125 DRIVER_CHECK_SVERSION + DRIVER_CHECK_HWTYPE,
126 MODULE_NAME, "I/O MMU", (void *) sba_driver_callback},
127
128 #if 0
129 /* FIXME : N-class! Use a different "callback"? */
130 {HPHW_BCPORT, IKE_MERCED_PORT, 0x0, 0xb, 0, 0x10,
131 DRIVER_CHECK_HVERSION +
132 DRIVER_CHECK_SVERSION + DRIVER_CHECK_HWTYPE,
133 MODULE_NAME, "I/O MMU", (void *) sba_driver_callback},
134
135 {HPHW_BCPORT, IKE_ROPES_PORT, 0x0, 0xb, 0, 0x10,
136 DRIVER_CHECK_HVERSION +
137 DRIVER_CHECK_SVERSION + DRIVER_CHECK_HWTYPE,
138 MODULE_NAME, "I/O MMU", (void *) sba_driver_callback},
139 #endif
140
141 {0,0,0,0,0,0,
142 0,
143 (char *) NULL, (char *) NULL, (void *) NULL }
144 };
145
146
147 #define SBA_FUNC_ID 0x0000 /* function id */
148 #define SBA_FCLASS 0x0008 /* function class, bist, header, rev... */
149
150 #define IS_ASTRO(id) ( \
151 (((id)->hw_type == HPHW_IOA) && ((id)->hversion == ASTRO_RUNWAY_PORT)) || \
152 (((id)->hw_type == HPHW_BCPORT) && ((id)->hversion == ASTRO_ROPES_PORT)) \
153 )
154
155 #define CONFIG_FUNC_SIZE 4096 /* SBA configuration function reg set */
156
157 #define ASTRO_IOC_OFFSET 0x20000
158 /* Ike's IOC's occupy functions 2 and 3 (not 0 and 1) */
159 #define IKE_IOC_OFFSET(p) ((p+2)*CONFIG_FUNC_SIZE)
160
161 #define IOC_CTRL 0x8 /* IOC_CTRL offset */
162 #define IOC_CTRL_TE (0x1 << 0) /* TOC Enable */
163 #define IOC_CTRL_RM (0x1 << 8) /* Real Mode */
164 #define IOC_CTRL_NC (0x1 << 9) /* Non Coherent Mode */
165
166 #define MAX_IOC 2 /* per Ike. Astro only has 1 */
167
168
169 /*
170 ** Offsets into MBIB (Function 0 on Ike and hopefully Astro)
171 ** Firmware programs this stuff. Don't touch it.
172 */
173 #define IOS_DIST_BASE 0x390
174 #define IOS_DIST_MASK 0x398
175 #define IOS_DIST_ROUTE 0x3A0
176
177 #define IOS_DIRECT_BASE 0x3C0
178 #define IOS_DIRECT_MASK 0x3C8
179 #define IOS_DIRECT_ROUTE 0x3D0
180
181 /*
182 ** Offsets into I/O TLB (Function 2 and 3 on Ike)
183 */
184 #define ROPE0_CTL 0x200 /* "regbus pci0" */
185 #define ROPE1_CTL 0x208
186 #define ROPE2_CTL 0x210
187 #define ROPE3_CTL 0x218
188 #define ROPE4_CTL 0x220
189 #define ROPE5_CTL 0x228
190 #define ROPE6_CTL 0x230
191 #define ROPE7_CTL 0x238
192
193 #define HF_ENABLE 0x40
194
195
196 #define IOC_IBASE 0x300 /* IO TLB */
197 #define IOC_IMASK 0x308
198 #define IOC_PCOM 0x310
199 #define IOC_TCNFG 0x318
200 #define IOC_PDIR_BASE 0x320
201
202 #define IOC_IOVA_SPACE_BASE 0 /* IOVA ranges start at 0 */
203
204 /*
205 ** IOC supports 4/8/16/64KB page sizes (see TCNFG register)
206 ** It's safer (avoid memory corruption) to keep DMA page mappings
207 ** equivalently sized to VM PAGE_SIZE.
208 **
209 ** We really can't avoid generating a new mapping for each
210 ** page since the Virtual Coherence Index has to be generated
211 ** and updated for each page.
212 **
213 ** IOVP_SIZE could only be greater than PAGE_SIZE if we are
214 ** confident the drivers really only touch the next physical
215 ** page iff that driver instance owns it.
216 */
217 #define IOVP_SIZE PAGE_SIZE
218 #define IOVP_SHIFT PAGE_SHIFT
219 #define IOVP_MASK PAGE_MASK
220
221 #define SBA_PERF_CFG 0x708 /* Performance Counter stuff */
222 #define SBA_PERF_MASK1 0x718
223 #define SBA_PERF_MASK2 0x730
224
225
226 /*
227 ** Offsets into PCI Performance Counters (functions 12 and 13)
228 ** Controlled by PERF registers in function 2 & 3 respectively.
229 */
230 #define SBA_PERF_CNT1 0x200
231 #define SBA_PERF_CNT2 0x208
232 #define SBA_PERF_CNT3 0x210
233
234
235 struct ioc {
236 char *ioc_hpa; /* I/O MMU base address */
237 char *res_map; /* resource map, bit == pdir entry */
238 u64 *pdir_base; /* physical base address */
239
240 unsigned long *res_hint; /* next available IOVP - circular search */
241 unsigned int res_bitshift; /* from the LEFT! */
242 unsigned int res_size; /* size of resource map in bytes */
243 unsigned int hint_shift_pdir;
244 spinlock_t res_lock;
245 unsigned long hint_mask_pdir; /* bits used for DMA hints */
246 #ifdef DELAYED_RESOURCE_CNT
247 dma_addr_t res_delay[DELAYED_RESOURCE_CNT];
248 #endif
249
250 #ifdef CONFIG_PROC_FS
251 #define SBA_SEARCH_SAMPLE 0x100
252 unsigned long avg_search[SBA_SEARCH_SAMPLE];
253 unsigned long avg_idx; /* current index into avg_search */
254 unsigned long used_pages;
255 unsigned long msingle_calls;
256 unsigned long msingle_pages;
257 unsigned long msg_calls;
258 unsigned long msg_pages;
259 unsigned long usingle_calls;
260 unsigned long usingle_pages;
261 unsigned long usg_calls;
262 unsigned long usg_pages;
263 #endif
264
265 /* STUFF We don't need in performance path */
266 unsigned int pdir_size; /* in bytes, determined by IOV Space size */
267 unsigned long ibase; /* pdir IOV Space base - shared w/lba_pci */
268 unsigned long imask; /* pdir IOV Space mask - shared w/lba_pci */
269 };
270
271 struct sba_device {
272 struct sba_device *next; /* list of LBA's in system */
273 struct hp_device *iodc; /* data about dev from firmware */
274 char *sba_hpa; /* base address */
275 spinlock_t sba_lock;
276 unsigned int flags; /* state/functionality enabled */
277 unsigned int hw_rev; /* HW revision of chip */
278
279 unsigned int num_ioc; /* number of on-board IOC's */
280 struct ioc ioc[MAX_IOC];
281 };
282
283
284 static struct sba_device *sba_list;
285 static int sba_count;
286
287 /* Ratio of Host MEM to IOV Space size */
288 static unsigned long sba_mem_ratio = 4;
289
290 /* Looks nice and keeps the compiler happy */
291 #define SBA_DEV(d) ((struct sba_device *) (d))
292
293
294 #define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
295
296
297 /************************************
298 ** SBA register read and write support
299 **
300 ** BE WARNED: register writes are posted.
301 ** (ie follow writes which must reach HW with a read)
302 */
303 #define READ_U8(addr) gsc_readb(addr)
304 #define READ_U16(addr) gsc_readw((u16 *) (addr))
305 #define READ_U32(addr) gsc_readl((u32 *) (addr))
306 #define WRITE_U8(value, addr) gsc_writeb(value, addr)
307 #define WRITE_U16(value, addr) gsc_writew(value, (u16 *) (addr))
308 #define WRITE_U32(value, addr) gsc_writel(value, (u32 *) (addr))
309
310 #define READ_REG8(addr) gsc_readb(addr)
311 #define READ_REG16(addr) le16_to_cpu(gsc_readw((u16 *) (addr)))
312 #define READ_REG32(addr) le32_to_cpu(gsc_readl((u32 *) (addr)))
313 #define READ_REG64(addr) le64_to_cpu(gsc_readq((u64 *) (addr)))
314 #define WRITE_REG8(value, addr) gsc_writeb(value, addr)
315 #define WRITE_REG16(value, addr) gsc_writew(cpu_to_le16(value), (u16 *) (addr))
316 #define WRITE_REG32(value, addr) gsc_writel(cpu_to_le32(value), (u32 *) (addr))
317 #define WRITE_REG64(value, addr) gsc_writeq(cpu_to_le64(value), (u64 *) (addr))
318
319 #ifdef DEBUG_SBA_INIT
320
321 static void
322 sba_dump_ranges(char *hpa)
323 {
324 printk("SBA at 0x%p\n", hpa);
325 printk("IOS_DIST_BASE : %08x %08x\n",
326 READ_REG32(hpa+IOS_DIST_BASE+4),
327 READ_REG32(hpa+IOS_DIST_BASE));
328 printk("IOS_DIST_MASK : %08x %08x\n",
329 READ_REG32(hpa+IOS_DIST_MASK+4),
330 READ_REG32(hpa+IOS_DIST_MASK));
331 printk("IOS_DIST_ROUTE : %08x %08x\n",
332 READ_REG32(hpa+IOS_DIST_ROUTE+4),
333 READ_REG32(hpa+IOS_DIST_ROUTE));
334 printk("\n");
335 printk("IOS_DIRECT_BASE : %08x %08x\n",
336 READ_REG32(hpa+IOS_DIRECT_BASE+4),
337 READ_REG32(hpa+IOS_DIRECT_BASE));
338 printk("IOS_DIRECT_MASK : %08x %08x\n",
339 READ_REG32(hpa+IOS_DIRECT_MASK+4),
340 READ_REG32(hpa+IOS_DIRECT_MASK));
341 printk("IOS_DIRECT_ROUTE: %08x %08x\n",
342 READ_REG32(hpa+IOS_DIRECT_ROUTE+4),
343 READ_REG32(hpa+IOS_DIRECT_ROUTE));
344 }
345
346 static void
347 sba_dump_tlb(char *hpa)
348 {
349 printk("IO TLB at 0x%p\n", hpa);
350 printk("IOC_IBASE : %08x %08x\n",
351 READ_REG32(hpa+IOC_IBASE+4),
352 READ_REG32(hpa+IOC_IBASE));
353 printk("IOC_IMASK : %08x %08x\n",
354 READ_REG32(hpa+IOC_IMASK+4),
355 READ_REG32(hpa+IOC_IMASK));
356 printk("IOC_TCNFG : %08x %08x\n",
357 READ_REG32(hpa+IOC_TCNFG+4),
358 READ_REG32(hpa+IOC_TCNFG));
359 printk("IOC_PDIR_BASE: %08x %08x\n",
360 READ_REG32(hpa+IOC_PDIR_BASE+4),
361 READ_REG32(hpa+IOC_PDIR_BASE));
362 printk("\n");
363 }
364 #endif
365
366
367 #ifdef ASSERT_PDIR_SANITY
368
369 static void
370 sba_dump_pdir_entry(struct ioc *ioc, char *msg, uint pide)
371 {
372 /* start printing from lowest pde in rval */
373 u64 *ptr = &(ioc->pdir_base[pide & (~0U * BITS_PER_LONG)]);
374 unsigned long *rptr = (unsigned long *) &(ioc->res_map[(pide >>3) & ~(sizeof(unsigned long) - 1)]);
375 uint rcnt;
376
377 printk("SBA: %s rp %p bit %d rval 0x%lx\n",
378 msg,
379 rptr, pide & (BITS_PER_LONG - 1), *rptr);
380
381 rcnt = 0;
382 while (rcnt < BITS_PER_LONG) {
383 printk("%s %2d %p %016Lx\n",
384 (rcnt == (pide & (BITS_PER_LONG - 1)))
385 ? " -->" : " ",
386 rcnt, ptr, *ptr );
387 rcnt++;
388 ptr++;
389 }
390 printk(msg);
391 }
392
393
394 /* Verify the resource map and pdir state is consistent */
395 static int
396 sba_check_pdir(struct ioc *ioc, char *msg)
397 {
398 u32 *rptr_end = (u32 *) &(ioc->res_map[ioc->res_size]);
399 u32 *rptr = (u32 *) ioc->res_map; /* resource map ptr */
400 u64 *pptr = ioc->pdir_base; /* pdir ptr */
401 uint pide = 0;
402
403 while (rptr < rptr_end) {
404 u32 rval = *rptr;
405 int rcnt = 32; /* number of bits we might check */
406
407 while (rcnt) {
408 /* Get last byte and highest bit from that */
409 u32 pde = ((u32) (((char *)pptr)[7])) << 24;
410 if ((rval ^ pde) & 0x80000000)
411 {
412 /*
413 ** BUMMER! -- res_map != pdir --
414 ** Dump rval and matching pdir entries
415 */
416 sba_dump_pdir_entry(ioc, msg, pide);
417 return(1);
418 }
419 rcnt--;
420 rval <<= 1; /* try the next bit */
421 pptr++;
422 pide++;
423 }
424 rptr++; /* look at next word of res_map */
425 }
426 /* It'd be nice if we always got here :^) */
427 return 0;
428 }
429
430
431 static void
432 sba_dump_sg( struct ioc *ioc, struct scatterlist *startsg, int nents)
433 {
434 while (nents-- > 0) {
435 printk(" %d : %08lx/%05x %p/%05x\n",
436 nents,
437 (unsigned long) sg_dma_address(startsg),
438 sg_dma_len(startsg),
439 startsg->address, startsg->length);
440 startsg++;
441 }
442 }
443
444 #endif /* ASSERT_PDIR_SANITY */
445
446
447
448 /*
449 ** One time initialization to let the world know the LBA was found.
450 ** This is the only routine which is NOT static.
451 ** Must be called exactly once before pci_init().
452 */
453 void __init
454 sba_init(void)
455 {
456 sba_list = (struct sba_device *) NULL;
457 sba_count = 0;
458
459 #ifdef DEBUG_SBA_INIT
460 sba_dump_ranges((char *) 0xFED00000L);
461 #endif
462
463 register_driver(sba_drivers_for);
464 }
465
466
467
468 /**************************************************************
469 *
470 * I/O Pdir Resource Management
471 *
472 * Bits set in the resource map are in use.
473 * Each bit can represent a number of pages.
474 * LSbs represent lower addresses (IOVA's).
475 *
476 ***************************************************************/
477 #define PAGES_PER_RANGE 1 /* could increase this to 4 or 8 if needed */
478
479 /* Convert from IOVP to IOVA and vice versa. */
480 #define SBA_IOVA(ioc,iovp,offset,hint_reg) ((iovp) | (offset) | ((hint_reg)<<(ioc->hint_shift_pdir)))
481 #define SBA_IOVP(ioc,iova) ((iova) & ioc->hint_mask_pdir)
482
483 /* FIXME : review these macros to verify correctness and usage */
484 #define PDIR_INDEX(iovp) ((iovp)>>IOVP_SHIFT)
485 #define MKIOVP(dma_hint,pide) (dma_addr_t)((long)(dma_hint) | ((long)(pide) << IOVP_SHIFT))
486 #define MKIOVA(iovp,offset) (dma_addr_t)((long)iovp | (long)offset)
487
488 #define RESMAP_MASK(n) (~0UL << (BITS_PER_LONG - (n)))
489 #define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
490
491
492 /*
493 ** Perf optimizations:
494 ** o search for log2(size) bits at a time.
495 **
496 ** Search should use register width as "stride" to search the res_map.
497 */
498
499 static SBA_INLINE unsigned long
500 sba_search_bitmap(struct ioc *ioc, unsigned long bits_wanted)
501 {
502 unsigned long *res_ptr = ioc->res_hint;
503 unsigned long *res_end = (unsigned long *) &(ioc->res_map[ioc->res_size]);
504 unsigned long pide = ~0UL;
505
506 ASSERT(((unsigned long) ioc->res_hint & (sizeof(unsigned long) - 1UL)) == 0);
507 ASSERT(res_ptr < res_end);
508 if (bits_wanted > (BITS_PER_LONG/2)) {
509 /* Search word at a time - no mask needed */
510 for(; res_ptr < res_end; ++res_ptr) {
511 if (*res_ptr == 0) {
512 *res_ptr = RESMAP_MASK(bits_wanted);
513 pide = ((unsigned long)res_ptr - (unsigned long)ioc->res_map);
514 pide <<= 3; /* convert to bit address */
515 ASSERT(0 != pide);
516 break;
517 }
518 }
519 /* point to the next word on next pass */
520 res_ptr++;
521 ioc->res_bitshift = 0;
522 } else {
523 /*
524 ** Search the resource bit map on well-aligned values.
525 ** "o" is the alignment.
526 ** We need the alignment to invalidate I/O TLB using
527 ** SBA HW features in the unmap path.
528 */
529 unsigned long o = 1 << get_order(bits_wanted << PAGE_SHIFT);
530 uint bitshiftcnt = ROUNDUP(ioc->res_bitshift, o);
531 unsigned long mask;
532
533 if (bitshiftcnt >= BITS_PER_LONG) {
534 bitshiftcnt = 0;
535 res_ptr++;
536 }
537 mask = RESMAP_MASK(bits_wanted) >> bitshiftcnt;
538
539 DBG_RES("sba_search_bitmap() o %ld %p", o, res_ptr);
540 while(res_ptr < res_end)
541 {
542 DBG_RES(" %p %lx %lx\n", res_ptr, mask, *res_ptr);
543 ASSERT(0 != mask);
544 if(0 == ((*res_ptr) & mask)) {
545 *res_ptr |= mask; /* mark resources busy! */
546 pide = ((unsigned long)res_ptr - (unsigned long)ioc->res_map);
547 pide <<= 3; /* convert to bit address */
548 pide += bitshiftcnt;
549 ASSERT(0 != pide);
550 break;
551 }
552 mask >>= o;
553 bitshiftcnt += o;
554 if (0 == mask) {
555 mask = RESMAP_MASK(bits_wanted);
556 bitshiftcnt=0;
557 res_ptr++;
558 }
559 }
560 /* look in the same word on the next pass */
561 ioc->res_bitshift = bitshiftcnt + bits_wanted;
562 }
563
564 /* wrapped ? */
565 ioc->res_hint = (res_end == res_ptr) ? (unsigned long *) ioc->res_map : res_ptr;
566 return (pide);
567 }
568
569
570 static int
571 sba_alloc_range(struct ioc *ioc, size_t size)
572 {
573 unsigned int pages_needed = size >> IOVP_SHIFT;
574 #ifdef CONFIG_PROC_FS
575 unsigned long cr_start = mfctl(16);
576 #endif
577 unsigned long pide;
578
579 ASSERT(pages_needed);
580 ASSERT((pages_needed * IOVP_SIZE) < DMA_CHUNK_SIZE);
581 ASSERT(pages_needed < BITS_PER_LONG);
582 ASSERT(0 == (size & ~IOVP_MASK));
583
584 /*
585 ** "seek and ye shall find"...praying never hurts either...
586 ** ggg sacrifices another 710 to the computer gods.
587 */
588
589 pide = sba_search_bitmap(ioc, pages_needed);
590 if (pide >= (ioc->res_size << 3)) {
591 pide = sba_search_bitmap(ioc, pages_needed);
592 if (pide >= (ioc->res_size << 3))
593 panic(__FILE__ ": I/O MMU @ %p is out of mapping resources\n", ioc->ioc_hpa);
594 }
595
596 #ifdef ASSERT_PDIR_SANITY
597 /* verify the first enable bit is clear */
598 if(0x00 != ((u8 *) ioc->pdir_base)[pide*sizeof(u64) + 7]) {
599 sba_dump_pdir_entry(ioc, "sba_search_bitmap() botched it?", pide);
600 }
601 #endif
602
603 DBG_RES("sba_alloc_range(%x) %d -> %lx hint %x/%x\n",
604 size, pages_needed, pide,
605 (uint) ((unsigned long) ioc->res_hint - (unsigned long) ioc->res_map),
606 ioc->res_bitshift );
607
608 #ifdef CONFIG_PROC_FS
609 {
610 unsigned long cr_end = mfctl(16);
611 unsigned long tmp = cr_end - cr_start;
612 /* check for roll over */
613 cr_start = (cr_end < cr_start) ? -(tmp) : (tmp);
614 }
615 ioc->avg_search[ioc->avg_idx++] = cr_start;
616 ioc->avg_idx &= SBA_SEARCH_SAMPLE - 1;
617
618 ioc->used_pages += pages_needed;
619 #endif
620
621 return (pide);
622 }
623
624
625 /*
626 ** clear bits in the ioc's resource map
627 */
628 static SBA_INLINE void
629 sba_free_range(struct ioc *ioc, dma_addr_t iova, size_t size)
630 {
631 unsigned long iovp = SBA_IOVP(ioc, iova);
632 unsigned int pide = PDIR_INDEX(iovp);
633 unsigned int ridx = pide >> 3; /* convert bit to byte address */
634 unsigned long *res_ptr = (unsigned long *) &((ioc)->res_map[ridx & ~RESMAP_IDX_MASK]);
635
636 int bits_not_wanted = size >> IOVP_SHIFT;
637
638 /* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
639 unsigned long m = RESMAP_MASK(bits_not_wanted) >> (pide & (BITS_PER_LONG - 1));
640
641 DBG_RES("sba_free_range( ,%x,%x) %x/%lx %x %p %lx\n",
642 (uint) iova, size,
643 bits_not_wanted, m, pide, res_ptr, *res_ptr);
644
645 #ifdef CONFIG_PROC_FS
646 ioc->used_pages -= bits_not_wanted;
647 #endif
648
649 ASSERT(m != 0);
650 ASSERT(bits_not_wanted);
651 ASSERT((bits_not_wanted * IOVP_SIZE) < DMA_CHUNK_SIZE);
652 ASSERT(bits_not_wanted < BITS_PER_LONG);
653 ASSERT((*res_ptr & m) == m); /* verify same bits are set */
654 *res_ptr &= ~m;
655 }
656
657
658 /**************************************************************
659 *
660 * "Dynamic DMA Mapping" support (aka "Coherent I/O")
661 *
662 ***************************************************************/
663
664 #define SBA_DMA_HINT(ioc, val) ((val) << (ioc)->hint_shift_pdir)
665
666
667 typedef unsigned long space_t;
668 #define KERNEL_SPACE 0
669
670 /*
671 * SBA Mapping Routine
672 *
673 * Given a virtual address (vba, arg2) and space id, (sid, arg1)
674 * sba_io_pdir_entry() loads the I/O PDIR entry pointed to by
675 * pdir_ptr (arg0). Each IO Pdir entry consists of 8 bytes as
676 * shown below (MSB == bit 0):
677 *
678 * 0 19 51 55 63
679 * +-+---------------------+----------------------------------+----+--------+
680 * |V| U | PPN[43:12] | U | VI |
681 * +-+---------------------+----------------------------------+----+--------+
682 *
683 * V == Valid Bit
684 * U == Unused
685 * PPN == Physical Page Number
686 * VI == Virtual Index (aka Coherent Index)
687 *
688 * The physical address fields are filled with the results of the LPA
689 * instruction. The virtual index field is filled with the results of
690 * of the LCI (Load Coherence Index) instruction. The 8 bits used for
691 * the virtual index are bits 12:19 of the value returned by LCI.
692 *
693 * We need to pre-swap the bytes since PCX-W is Big Endian.
694 */
695
696 void SBA_INLINE
697 sba_io_pdir_entry(u64 *pdir_ptr, space_t sid, unsigned long vba)
698 {
699 u64 pa; /* physical address */
700 register unsigned ci; /* coherent index */
701
702 /* We currently only support kernel addresses */
703 ASSERT(sid == 0);
704 ASSERT(((unsigned long) vba & 0xc0000000UL) == 0xc0000000UL);
705
706 pa = virt_to_phys(vba);
707 pa &= ~4095ULL; /* clear out offset bits */
708
709 mtsp(sid,1);
710 asm("lci 0(%%sr1, %1), %0" : "=r" (ci) : "r" (vba));
711 pa |= (ci >> 12) & 0xff; /* move CI (8 bits) into lowest byte */
712
713 pa |= 0x8000000000000000ULL; /* set "valid" bit */
714 *pdir_ptr = cpu_to_le64(pa); /* swap and store into I/O Pdir */
715 }
716
717
718 /***********************************************************
719 * The Ike PCOM (Purge Command Register) is to purge
720 * stale entries in the IO TLB when unmapping entries.
721 *
722 * The PCOM register supports purging of multiple pages, with a minium
723 * of 1 page and a maximum of 2GB. Hardware requires the address be
724 * aligned to the size of the range being purged. The size of the range
725 * must be a power of 2.
726 ***********************************************************/
727 static SBA_INLINE void
728 sba_mark_invalid(struct ioc *ioc, dma_addr_t iova, size_t byte_cnt)
729 {
730 u32 iovp = (u32) SBA_IOVP(ioc,iova);
731
732 /* Even though this is a big-endian machine, the entries
733 ** in the iopdir are swapped. That's why we clear the byte
734 ** at +7 instead of at +0.
735 */
736 int off = PDIR_INDEX(iovp)*sizeof(u64)+7;
737
738 /* Must be non-zero and rounded up */
739 ASSERT(byte_cnt > 0);
740 ASSERT(0 == (byte_cnt & ~IOVP_MASK));
741
742 #ifdef ASSERT_PDIR_SANITY
743 /* Assert first pdir entry is set */
744 if (0x80 != (((u8 *) ioc->pdir_base)[off])) {
745 sba_dump_pdir_entry(ioc,"sba_mark_invalid()", PDIR_INDEX(iovp));
746 }
747 #endif
748
749 if (byte_cnt <= IOVP_SIZE)
750 {
751 ASSERT( off < ioc->pdir_size);
752
753 iovp |= IOVP_SHIFT; /* set "size" field for PCOM */
754
755 /*
756 ** clear I/O PDIR entry "valid" bit
757 ** Do NOT clear the rest - save it for debugging.
758 ** We should only clear bits that have previously
759 ** been enabled.
760 */
761 ((u8 *)(ioc->pdir_base))[off] = 0;
762 } else {
763 u32 t = get_order(byte_cnt) + PAGE_SHIFT;
764
765 iovp |= t;
766 ASSERT(t <= 31); /* 2GB! Max value of "size" field */
767
768 do {
769 /* verify this pdir entry is enabled */
770 ASSERT(0x80 == (((u8 *) ioc->pdir_base)[off] & 0x80));
771 /* clear I/O Pdir entry "valid" bit first */
772 ((u8 *)(ioc->pdir_base))[off] = 0;
773 off += sizeof(u64);
774 byte_cnt -= IOVP_SIZE;
775 } while (byte_cnt > 0);
776 }
777
778 WRITE_REG32(iovp, ioc->ioc_hpa+IOC_PCOM);
779 }
780
781 static int
782 sba_dma_supported( struct pci_dev *dev, dma_addr_t mask)
783 {
784 if (dev == NULL) {
785 printk(MODULE_NAME ": EISA/ISA/et al not supported\n");
786 BUG();
787 return(0);
788 }
789
790 dev->dma_mask = mask; /* save it */
791
792 /* only support PCI devices */
793 return((int) (mask >= 0xffffffff));
794 }
795
796
797 /*
798 ** map_single returns a fully formed IOVA
799 */
800 static dma_addr_t
801 sba_map_single(struct pci_dev *dev, void *addr, size_t size, int direction)
802 {
803 struct ioc *ioc = &sba_list->ioc[0]; /* FIXME : see Multi-IOC below */
804 unsigned long flags;
805 dma_addr_t iovp;
806 dma_addr_t offset;
807 u64 *pdir_start;
808 int pide;
809
810 ASSERT(size > 0);
811
812 /* save offset bits */
813 offset = ((dma_addr_t) addr) & ~IOVP_MASK;
814
815 /* round up to nearest IOVP_SIZE */
816 size = (size + offset + ~IOVP_MASK) & IOVP_MASK;
817
818 spin_lock_irqsave(&ioc->res_lock, flags);
819 #ifdef ASSERT_PDIR_SANITY
820 sba_check_pdir(ioc,"Check before sba_map_single()");
821 #endif
822
823 #ifdef CONFIG_PROC_FS
824 ioc->msingle_calls++;
825 ioc->msingle_pages += size >> IOVP_SHIFT;
826 #endif
827 pide = sba_alloc_range(ioc, size);
828 iovp = (dma_addr_t) pide << IOVP_SHIFT;
829
830 DBG_RUN("sba_map_single() 0x%p -> 0x%lx", addr, (long) iovp | offset);
831
832 pdir_start = &(ioc->pdir_base[pide]);
833
834 while (size > 0) {
835 ASSERT(((u8 *)pdir_start)[7] == 0); /* verify availability */
836 sba_io_pdir_entry(pdir_start, KERNEL_SPACE, (unsigned long) addr);
837
838 DBG_RUN(" pdir 0x%p %02x%02x%02x%02x%02x%02x%02x%02x\n",
839 pdir_start,
840 (u8) (((u8 *) pdir_start)[7]),
841 (u8) (((u8 *) pdir_start)[6]),
842 (u8) (((u8 *) pdir_start)[5]),
843 (u8) (((u8 *) pdir_start)[4]),
844 (u8) (((u8 *) pdir_start)[3]),
845 (u8) (((u8 *) pdir_start)[2]),
846 (u8) (((u8 *) pdir_start)[1]),
847 (u8) (((u8 *) pdir_start)[0])
848 );
849
850 addr += IOVP_SIZE;
851 size -= IOVP_SIZE;
852 pdir_start++;
853 }
854 /* form complete address */
855 #ifdef ASSERT_PDIR_SANITY
856 sba_check_pdir(ioc,"Check after sba_map_single()");
857 #endif
858 spin_unlock_irqrestore(&ioc->res_lock, flags);
859 return SBA_IOVA(ioc, iovp, offset, DEFAULT_DMA_HINT_REG);
860 }
861
862
863 static void
864 sba_unmap_single(struct pci_dev *dev, dma_addr_t iova, size_t size, int direction)
865 {
866 #ifdef FIXME
867 /* Multi-IOC (ie N-class) : need to lookup IOC from dev
868 ** o If we can't know about lba PCI data structs, that eliminates ->sysdata.
869 ** o walking up pcidev->parent dead ends at elroy too
870 ** o leaves hashing dev->bus->number into some lookup.
871 ** (may only work for N-class)
872 ** o use (struct pci_hba) and put fields in there for DMA.
873 ** (ioc and per device dma_hint.)
874 **
875 ** Last one seems the clearest and most promising.
876 ** sba_dma_supported() fill in those fields when the driver queries
877 ** the system for support.
878 */
879 struct ioc *ioc = (struct ioc *) ((struct pci_hba *) (dev->sysdata))->dma_data;
880 #else
881 struct ioc *ioc = &sba_list->ioc[0];
882 #endif
883
884 unsigned long flags;
885 dma_addr_t offset;
886 offset = iova & ~IOVP_MASK;
887
888 DBG_RUN("%s() iovp 0x%lx/%x\n", __FUNCTION__, (long) iova, size);
889
890 iova ^= offset; /* clear offset bits */
891 size += offset;
892 size = ROUNDUP(size, IOVP_SIZE);
893
894 ASSERT(0 != iova);
895
896 spin_lock_irqsave(&ioc->res_lock, flags);
897 #ifdef CONFIG_PROC_FS
898 ioc->usingle_calls++;
899 ioc->usingle_pages += size >> IOVP_SHIFT;
900 #endif
901 #ifdef DELAYED_RESOURCE_CNT
902 if (ioc->saved_cnt < DELAYED_RESOURCE_CNT) {
903 ioc->saved_iova[ioc->saved_cnt] = iova;
904 ioc->saved_size[ioc->saved_cnt] = size;
905 ioc_saved_cnt++;
906 } else {
907 do {
908 #endif
909 sba_mark_invalid(ioc, iova, size);
910 sba_free_range(ioc, iova, size);
911
912 #ifdef DELAYED_RESOURCE_CNT
913 ioc->saved_cnt--;
914 iova = ioc->saved_iova[ioc->saved_cnt];
915 size = ioc->saved_size[ioc->saved_cnt];
916 } while (ioc->saved_cnt)
917
918 /* flush purges */
919 (void) (volatile) READ_REG32(ioc->ioc_hpa+IOC_PCOM);
920 }
921 #else
922 /* flush purges */
923 READ_REG32(ioc->ioc_hpa+IOC_PCOM);
924 #endif
925 spin_unlock_irqrestore(&ioc->res_lock, flags);
926 }
927
928
929 static void *
930 sba_alloc_consistent(struct pci_dev *hwdev, size_t size, dma_addr_t *dma_handle)
931 {
932 void *ret;
933
934 if (!hwdev) {
935 /* only support PCI */
936 *dma_handle = 0;
937 return 0;
938 }
939
940 ret = (void *) __get_free_pages(GFP_ATOMIC, get_order(size));
941
942 if (ret) {
943 memset(ret, 0, size);
944 *dma_handle = sba_map_single(hwdev, ret, size, 0);
945 }
946
947 return ret;
948 }
949
950
951 static void
952 sba_free_consistent(struct pci_dev *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle)
953 {
954 sba_unmap_single(hwdev, dma_handle, size, 0);
955 free_pages((unsigned long) vaddr, get_order(size));
956 }
957
958 /*
959 ** Two address ranges are "virtually contiguous" iff:
960 ** 1) end of prev == start of next, or... append case
961 ** 3) end of next == start of prev prepend case
962 **
963 ** and they are DMA contiguous *iff*:
964 ** 2) end of prev and start of next are both on a page boundry
965 **
966 ** (shift left is a quick trick to mask off upper bits)
967 */
968 #define DMA_CONTIG(__X, __Y) \
969 (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - PAGE_SHIFT)) == 0UL)
970
971 /*
972 ** Assumption is two transactions are mutually exclusive.
973 ** ie both go to different parts of memory.
974 ** If both are true, then both transaction are on the same page.
975 */
976 #define DMA_SAME_PAGE(s1,e1,s2,e2) \
977 ( ((((s1) ^ (s2)) >> PAGE_SHIFT) == 0) \
978 && ((((e1) ^ (e2)) >> PAGE_SHIFT) == 0) )
979
980 /*
981 ** Since 0 is a valid pdir_base index value, can't use that
982 ** to determine if a value is valid or not. Use a flag to indicate
983 ** the SG list entry contains a valid pdir index.
984 */
985 #define PIDE_FLAG 0x80000000UL
986
987 #ifdef DEBUG_LARGE_SG_ENTRIES
988 int dump_run_sg = 0;
989 #endif
990
991 static SBA_INLINE int
992 sba_fill_pdir(
993 struct ioc *ioc,
994 struct scatterlist *startsg,
995 int nents)
996 {
997 struct scatterlist *dma_sg = startsg; /* pointer to current DMA */
998 int n_mappings = 0;
999 u64 *pdirp = 0;
1000 unsigned long dma_offset = 0;
1001
1002 dma_sg--;
1003 while (nents-- > 0) {
1004 int cnt = sg_dma_len(startsg);
1005 sg_dma_len(startsg) = 0;
1006
1007 #ifdef DEBUG_LARGE_SG_ENTRIES
1008 if (dump_run_sg)
1009 printk(" %d : %08lx/%05x %p/%05x\n",
1010 nents,
1011 (unsigned long) sg_dma_address(startsg), cnt,
1012 startsg->address, startsg->length
1013 );
1014 #else
1015 DBG_RUN_SG(" %d : %08lx/%05x %p/%05x\n",
1016 nents,
1017 (unsigned long) sg_dma_address(startsg), cnt,
1018 startsg->address, startsg->length
1019 );
1020 #endif
1021 /*
1022 ** Look for the start of a new DMA stream
1023 */
1024 if (sg_dma_address(startsg) & PIDE_FLAG) {
1025 u32 pide = sg_dma_address(startsg) & ~PIDE_FLAG;
1026 dma_offset = (unsigned long) pide & ~IOVP_MASK;
1027 pide >>= IOVP_SHIFT;
1028 pdirp = &(ioc->pdir_base[pide]);
1029 sg_dma_address(startsg) = 0;
1030 ++dma_sg;
1031 sg_dma_address(dma_sg) = (pide << IOVP_SHIFT) + dma_offset;
1032 n_mappings++;
1033 }
1034
1035 /*
1036 ** Look for a VCONTIG chunk
1037 */
1038 if (cnt) {
1039 unsigned long vaddr = (unsigned long) startsg->address;
1040 ASSERT(pdirp);
1041
1042 sg_dma_len(dma_sg) += cnt;
1043 cnt += dma_offset;
1044 dma_offset=0; /* only want offset on first chunk */
1045 cnt = ROUNDUP(cnt, IOVP_SIZE);
1046 #ifdef CONFIG_PROC_FS
1047 ioc->msg_pages += cnt >> IOVP_SHIFT;
1048 #endif
1049 do {
1050 sba_io_pdir_entry(pdirp, KERNEL_SPACE, vaddr);
1051 vaddr += IOVP_SIZE;
1052 cnt -= IOVP_SIZE;
1053 pdirp++;
1054 } while (cnt > 0);
1055 }
1056 startsg++;
1057 }
1058 #ifdef DEBUG_LARGE_SG_ENTRIES
1059 dump_run_sg = 0;
1060 #endif
1061 return(n_mappings);
1062 }
1063
1064
1065
1066 /*
1067 ** First pass is to walk the SG list and determine where the breaks are
1068 ** in the DMA stream. Allocates PDIR entries but does not fill them.
1069 ** Returns the number of DMA chunks.
1070 **
1071 ** Doing the fill seperate from the coalescing/allocation keeps the
1072 ** code simpler. Future enhancement could make one pass through
1073 ** the sglist do both.
1074 */
1075 static SBA_INLINE int
1076 sba_coalesce_chunks( struct ioc *ioc,
1077 struct scatterlist *startsg,
1078 int nents)
1079 {
1080 int n_mappings = 0;
1081
1082 while (nents > 0) {
1083 struct scatterlist *dma_sg; /* next DMA stream head */
1084 unsigned long dma_offset, dma_len; /* start/len of DMA stream */
1085 struct scatterlist *chunksg; /* virtually contig chunk head */
1086 unsigned long chunk_addr, chunk_len; /* start/len of VCONTIG chunk */
1087
1088 /*
1089 ** Prepare for first/next DMA stream
1090 */
1091 dma_sg = chunksg = startsg;
1092 dma_len = chunk_len = startsg->length;
1093 chunk_addr = (unsigned long) startsg->address;
1094 dma_offset = 0UL;
1095
1096 /*
1097 ** This loop terminates one iteration "early" since
1098 ** it's always looking one "ahead".
1099 */
1100 while (--nents > 0) {
1101 /* ptr to coalesce prev and next */
1102 struct scatterlist *prev_sg = startsg;
1103 unsigned long prev_end = (unsigned long) prev_sg->address + prev_sg->length;
1104 unsigned long current_end;
1105
1106 /* PARANOID: clear entries */
1107 sg_dma_address(startsg) = 0;
1108 sg_dma_len(startsg) = 0;
1109
1110 /* Now start looking ahead */
1111 startsg++;
1112 current_end = (unsigned long) startsg->address + startsg->length;
1113
1114 /*
1115 ** First look for virtually contiguous blocks.
1116 ** PARISC needs this since it's cache is virtually
1117 ** indexed and we need the associated virtual
1118 ** address for each I/O address we map.
1119 **
1120 ** 1) can we *prepend* the next transaction?
1121 */
1122 if (current_end == (unsigned long) prev_sg->address)
1123 {
1124 /* prepend : get new offset */
1125 chunksg = startsg;
1126 chunk_addr = (unsigned long) prev_sg->address;
1127 chunk_len += startsg->length;
1128 dma_len += startsg->length;
1129 continue;
1130 }
1131
1132 /*
1133 ** 2) or append the next transaction?
1134 */
1135 if (prev_end == (unsigned long) startsg->address)
1136 {
1137 chunk_len += startsg->length;
1138 dma_len += startsg->length;
1139 continue;
1140 }
1141
1142 #ifdef DEBUG_LARGE_SG_ENTRIES
1143 dump_run_sg = (chunk_len > IOVP_SIZE);
1144 #endif
1145 /*
1146 ** Not virtually contigous.
1147 ** Terminate prev chunk.
1148 ** Start a new chunk.
1149 **
1150 ** Once we start a new VCONTIG chunk, the offset
1151 ** can't change. And we need the offset from the first
1152 ** chunk - not the last one. Ergo Successive chunks
1153 ** must start on page boundaries and dove tail
1154 ** with it's predecessor.
1155 */
1156 sg_dma_len(prev_sg) = chunk_len;
1157
1158 chunk_len = startsg->length;
1159 dma_offset |= (chunk_addr & ~IOVP_MASK);
1160 ASSERT((0 == (chunk_addr & ~IOVP_MASK)) ||
1161 (dma_offset == (chunk_addr & ~IOVP_MASK)));
1162
1163 #if 0
1164 /*
1165 ** 4) do the chunks end/start on page boundaries?
1166 ** Easier than 3 since no offsets are involved.
1167 */
1168 if (DMA_CONTIG(prev_end, startsg->address))
1169 {
1170 /*
1171 ** Yes.
1172 ** Reset chunk ptr.
1173 */
1174 chunksg = startsg;
1175 chunk_addr = (unsigned long) startsg->address;
1176
1177 continue;
1178 } else
1179 #endif
1180 {
1181 break;
1182 }
1183 }
1184
1185 /*
1186 ** End of DMA Stream
1187 ** Terminate chunk.
1188 ** Allocate space for DMA stream.
1189 */
1190 sg_dma_len(startsg) = chunk_len;
1191 dma_len = (dma_len + dma_offset + ~IOVP_MASK) & IOVP_MASK;
1192 sg_dma_address(dma_sg) =
1193 PIDE_FLAG
1194 | (sba_alloc_range(ioc, dma_len) << IOVP_SHIFT)
1195 | dma_offset;
1196 n_mappings++;
1197 }
1198
1199 return n_mappings;
1200 }
1201
1202
1203 /*
1204 ** And this algorithm still generally only ends up coalescing entries
1205 ** that happens to be on the same page due to how sglists are assembled.
1206 */
1207 static int
1208 sba_map_sg(struct pci_dev *dev, struct scatterlist *sglist, int nents, int direction)
1209 {
1210 struct ioc *ioc = &sba_list->ioc[0]; /* FIXME : see Multi-IOC below */
1211 int coalesced, filled = 0;
1212 unsigned long flags;
1213
1214 DBG_RUN_SG("%s() START %d entries\n", __FUNCTION__, nents);
1215
1216 /* Fast path single entry scatterlists. */
1217 if (nents == 1) {
1218 sg_dma_address(sglist)= sba_map_single(dev, sglist->address,
1219 sglist->length, direction);
1220 sg_dma_len(sglist)= sglist->length;
1221 return 1;
1222 }
1223
1224 spin_lock_irqsave(&ioc->res_lock, flags);
1225
1226 #ifdef ASSERT_PDIR_SANITY
1227 if (sba_check_pdir(ioc,"Check before sba_map_sg()"))
1228 {
1229 sba_dump_sg(ioc, sglist, nents);
1230 panic("Check before sba_map_sg()");
1231 }
1232 #endif
1233
1234 #ifdef CONFIG_PROC_FS
1235 ioc->msg_calls++;
1236 #endif
1237
1238 /*
1239 ** First coalesce the chunks and allocate I/O pdir space
1240 **
1241 ** If this is one DMA stream, we can properly map using the
1242 ** correct virtual address associated with each DMA page.
1243 ** w/o this association, we wouldn't have coherent DMA!
1244 ** Access to the virtual address is what forces a two pass algorithm.
1245 */
1246 coalesced = sba_coalesce_chunks(ioc, sglist, nents);
1247
1248 /*
1249 ** Program the I/O Pdir
1250 **
1251 ** map the virtual addresses to the I/O Pdir
1252 ** o dma_address will contain the pdir index
1253 ** o dma_len will contain the number of bytes to map
1254 ** o address contains the virtual address.
1255 */
1256 filled = sba_fill_pdir(ioc, sglist, nents);
1257
1258 #ifdef ASSERT_PDIR_SANITY
1259 if (sba_check_pdir(ioc,"Check after sba_map_sg()"))
1260 {
1261 sba_dump_sg(ioc, sglist, nents);
1262 panic("Check after sba_map_sg()\n");
1263 }
1264 #endif
1265
1266 spin_unlock_irqrestore(&ioc->res_lock, flags);
1267
1268 ASSERT(coalesced == filled);
1269 DBG_RUN_SG("%s() DONE %d mappings\n", __FUNCTION__, filled);
1270
1271 return filled;
1272 }
1273
1274
1275 static void
1276 sba_unmap_sg(struct pci_dev *dev, struct scatterlist *sglist, int nents, int direction)
1277 {
1278 struct ioc *ioc = &sba_list->ioc[0]; /* FIXME : see Multi-IOC below */
1279 #ifdef ASSERT_PDIR_SANITY
1280 unsigned long flags;
1281 #endif
1282
1283 DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1284 __FUNCTION__, nents, sglist->address, sglist->length);
1285
1286 #ifdef CONFIG_PROC_FS
1287 ioc->usg_calls++;
1288 #endif
1289
1290 #ifdef ASSERT_PDIR_SANITY
1291 spin_lock_irqsave(&ioc->res_lock, flags);
1292 sba_check_pdir(ioc,"Check before sba_unmap_sg()");
1293 spin_unlock_irqrestore(&ioc->res_lock, flags);
1294 #endif
1295
1296 while (sg_dma_len(sglist) && nents--) {
1297
1298 #ifdef CONFIG_PROC_FS
1299 ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT;
1300 #endif
1301 sba_unmap_single(dev, sg_dma_address(sglist), sg_dma_len(sglist), direction);
1302 ++sglist;
1303 }
1304
1305 DBG_RUN_SG("%s() DONE (nents %d)\n", __FUNCTION__, nents);
1306
1307 #ifdef ASSERT_PDIR_SANITY
1308 spin_lock_irqsave(&ioc->res_lock, flags);
1309 sba_check_pdir(ioc,"Check after sba_unmap_sg()");
1310 spin_unlock_irqrestore(&ioc->res_lock, flags);
1311 #endif
1312
1313 }
1314
1315 static struct pci_dma_ops sba_ops = {
1316 sba_dma_supported,
1317 sba_alloc_consistent, /* allocate cacheable host mem */
1318 sba_free_consistent, /* release cacheable host mem */
1319 sba_map_single,
1320 sba_unmap_single,
1321 sba_map_sg,
1322 sba_unmap_sg,
1323 NULL, /* dma_sync_single */
1324 NULL /* dma_sync_sg */
1325 };
1326
1327
1328 /**************************************************************************
1329 **
1330 ** SBA PAT PDC support
1331 **
1332 ** o call pdc_pat_cell_module()
1333 ** o store ranges in PCI "resource" structures
1334 **
1335 **************************************************************************/
1336
1337 static void
1338 sba_get_pat_resources(struct sba_device *sba_dev)
1339 {
1340 #if 0
1341 /*
1342 ** TODO/REVISIT/FIXME: support for directed ranges requires calls to
1343 ** PAT PDC to program the SBA/LBA directed range registers...this
1344 ** burden may fall on the LBA code since it directly supports the
1345 ** PCI subsystem. It's not clear yet. - ggg
1346 */
1347 PAT_MOD(mod)->mod_info.mod_pages = PAT_GET_MOD_PAGES(temp);
1348 FIXME : ???
1349 PAT_MOD(mod)->mod_info.dvi = PAT_GET_DVI(temp);
1350 Tells where the dvi bits are located in the address.
1351 PAT_MOD(mod)->mod_info.ioc = PAT_GET_IOC(temp);
1352 FIXME : ???
1353 #endif
1354 }
1355
1356
1357 /**************************************************************
1358 *
1359 * Initialization and claim
1360 *
1361 ***************************************************************/
1362
1363
1364 static void
1365 sba_ioc_init(struct ioc *ioc)
1366 {
1367 extern unsigned long mem_max; /* arch.../setup.c */
1368 extern void lba_init_iregs(void *, u32, u32); /* arch.../lba_pci.c */
1369
1370 u32 iova_space_size, iova_space_mask;
1371 void * pdir_base;
1372 int pdir_size, iov_order;
1373
1374 /*
1375 ** Determine IOVA Space size from memory size.
1376 ** Using "mem_max" is a kluge.
1377 **
1378 ** Ideally, PCI drivers would register the maximum number
1379 ** of DMA they can have outstanding for each device they
1380 ** own. Next best thing would be to guess how much DMA
1381 ** can be outstanding based on PCI Class/sub-class. Both
1382 ** methods still require some "extra" to support PCI
1383 ** Hot-Plug/Removal of PCI cards. (aka PCI OLARD).
1384 **
1385 ** While we have 32-bits "IOVA" space, top two 2 bits are used
1386 ** for DMA hints - ergo only 30 bits max.
1387 */
1388 /* limit IOVA space size to 1MB-1GB */
1389 if (mem_max < (sba_mem_ratio*1024*1024)) {
1390 iova_space_size = 1024*1024;
1391 #ifdef __LP64__
1392 } else if (mem_max > (sba_mem_ratio*512*1024*1024)) {
1393 iova_space_size = 512*1024*1024;
1394 #endif
1395 } else {
1396 iova_space_size = (u32) (mem_max/sba_mem_ratio);
1397 }
1398
1399 /*
1400 ** iova space must be log2() in size.
1401 ** thus, pdir/res_map will also be log2().
1402 */
1403 iov_order = get_order(iova_space_size >> (IOVP_SHIFT-PAGE_SHIFT));
1404 ASSERT(iov_order <= (30 - IOVP_SHIFT)); /* iova_space_size <= 1GB */
1405 ASSERT(iov_order >= (20 - IOVP_SHIFT)); /* iova_space_size >= 1MB */
1406 iova_space_size = 1 << (iov_order + IOVP_SHIFT);
1407
1408 ioc->pdir_size = pdir_size = (iova_space_size/IOVP_SIZE) * sizeof(u64);
1409
1410 ASSERT(pdir_size < 4*1024*1024); /* max pdir size < 4MB */
1411
1412 /* Verify it's a power of two */
1413 ASSERT((1 << get_order(pdir_size)) == (pdir_size >> PAGE_SHIFT));
1414
1415 DBG_INIT("%s() hpa 0x%p mem %dMBIOV %dMB (%d bits) PDIR size 0x%0x",
1416 __FUNCTION__, ioc->ioc_hpa, (int) (mem_max>>20),
1417 iova_space_size>>20, iov_order + PAGE_SHIFT, pdir_size);
1418
1419 /* FIXME : DMA HINTs not used */
1420 ioc->hint_shift_pdir = iov_order + PAGE_SHIFT;
1421 ioc->hint_mask_pdir = ~(0x3 << (iov_order + PAGE_SHIFT));
1422
1423 ioc->pdir_base =
1424 pdir_base = (void *) __get_free_pages(GFP_KERNEL, get_order(pdir_size));
1425 if (NULL == pdir_base)
1426 {
1427 panic(__FILE__ ":%s() could not allocate I/O Page Table\n", __FUNCTION__);
1428 }
1429 memset(pdir_base, 0, pdir_size);
1430
1431 DBG_INIT("sba_ioc_init() pdir %p size %x hint_shift_pdir %x hint_mask_pdir %lx\n",
1432 pdir_base, pdir_size,
1433 ioc->hint_shift_pdir, ioc->hint_mask_pdir);
1434
1435 ASSERT((((unsigned long) pdir_base) & PAGE_MASK) == (unsigned long) pdir_base);
1436 WRITE_REG64(virt_to_phys(pdir_base), (u64 *)(ioc->ioc_hpa+IOC_PDIR_BASE));
1437
1438 DBG_INIT(" base %p\n", pdir_base);
1439
1440 /* build IMASK for IOC and Elroy */
1441 iova_space_mask = 0xffffffff;
1442 iova_space_mask <<= (iov_order + PAGE_SHIFT);
1443
1444 /*
1445 ** On C3000 w/512MB mem, HP-UX 10.20 reports:
1446 ** ibase=0, imask=0xFE000000, size=0x2000000.
1447 */
1448 ioc->ibase = IOC_IOVA_SPACE_BASE | 1; /* bit 0 == enable bit */
1449 ioc->imask = iova_space_mask; /* save it */
1450
1451 DBG_INIT("%s() IOV base 0x%lx mask 0x%0lx\n", __FUNCTION__,
1452 ioc->ibase, ioc->imask);
1453
1454 /*
1455 ** FIXME: Hint registers are programmed with default hint
1456 ** values during boot, so hints should be sane even if we
1457 ** can't reprogram them the way drivers want.
1458 */
1459
1460 /*
1461 ** setup Elroy IBASE/IMASK registers as well.
1462 */
1463 lba_init_iregs(ioc->ioc_hpa, ioc->ibase, ioc->imask);
1464
1465 /*
1466 ** Program the IOC's ibase and enable IOVA translation
1467 */
1468 WRITE_REG32(ioc->ibase, ioc->ioc_hpa+IOC_IBASE);
1469 WRITE_REG32(ioc->imask, ioc->ioc_hpa+IOC_IMASK);
1470
1471 /* Set I/O PDIR Page size to 4K */
1472 WRITE_REG32(0, ioc->ioc_hpa+IOC_TCNFG);
1473
1474 /*
1475 ** Clear I/O TLB of any possible entries.
1476 ** (Yes. This is a it paranoid...but so what)
1477 */
1478 WRITE_REG32(0 | 31, ioc->ioc_hpa+IOC_PCOM);
1479
1480 DBG_INIT("%s() DONE\n", __FUNCTION__);
1481 }
1482
1483
1484
1485 /**************************************************************************
1486 **
1487 ** SBA initialization code (HW and SW)
1488 **
1489 ** o identify SBA chip itself
1490 ** o initialize SBA chip modes (HardFail)
1491 ** o initialize SBA chip modes (HardFail)
1492 ** o FIXME: initialize DMA hints for reasonable defaults
1493 **
1494 **************************************************************************/
1495
1496 static void
1497 sba_hw_init(struct sba_device *sba_dev)
1498 {
1499 int i;
1500 int num_ioc;
1501 u32 ioc_ctl;
1502
1503 ioc_ctl = READ_REG32(sba_dev->sba_hpa+IOC_CTRL);
1504 DBG_INIT("%s() hpa 0x%p ioc_ctl 0x%x ->", __FUNCTION__, sba_dev->sba_hpa, ioc_ctl );
1505 ioc_ctl &= ~(IOC_CTRL_RM | IOC_CTRL_NC);
1506 ASSERT(ioc_ctl & IOC_CTRL_TE); /* astro: firmware enables this */
1507
1508 WRITE_REG32(ioc_ctl, sba_dev->sba_hpa+IOC_CTRL);
1509
1510 #ifdef SBA_DEBUG_INIT
1511 ioc_ctl = READ_REG32(sba_dev->sba_hpa+IOC_CTRL);
1512 DBG_INIT(" 0x%x\n", ioc_ctl );
1513 #endif
1514
1515 if (IS_ASTRO(sba_dev->iodc)) {
1516 /* PAT_PDC (L-class) also reports the same goofy base */
1517 sba_dev->ioc[0].ioc_hpa = (char *) ASTRO_IOC_OFFSET;
1518 num_ioc = 1;
1519 } else {
1520 sba_dev->ioc[0].ioc_hpa = sba_dev->ioc[1].ioc_hpa = 0;
1521 num_ioc = 2;
1522 }
1523
1524 sba_dev->num_ioc = num_ioc;
1525 for( i = 0; i < num_ioc; i++)
1526 {
1527 (unsigned long) sba_dev->ioc[i].ioc_hpa += (unsigned long) sba_dev->sba_hpa + IKE_IOC_OFFSET(i);
1528
1529 /*
1530 ** Make sure the box crashes if we get any errors on a rope.
1531 */
1532 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE0_CTL);
1533 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE1_CTL);
1534 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE2_CTL);
1535 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE3_CTL);
1536 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE4_CTL);
1537 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE5_CTL);
1538 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE6_CTL);
1539 WRITE_REG32(HF_ENABLE, sba_dev->ioc[i].ioc_hpa + ROPE7_CTL);
1540
1541 /* flush out the writes */
1542 READ_REG32(sba_dev->ioc[i].ioc_hpa + ROPE7_CTL);
1543
1544 sba_ioc_init(&(sba_dev->ioc[i]));
1545 }
1546 }
1547
1548 static void
1549 sba_common_init(struct sba_device *sba_dev)
1550 {
1551 int i;
1552
1553 /* add this one to the head of the list (order doesn't matter)
1554 ** This will be useful for debugging - especially if we get coredumps
1555 */
1556 sba_dev->next = sba_list;
1557 sba_list = sba_dev;
1558 sba_count++;
1559
1560 for(i=0; i< sba_dev->num_ioc; i++) {
1561 int res_size;
1562 #ifdef CONFIG_DMB_TRAP
1563 extern void iterate_pages(unsigned long , unsigned long ,
1564 void (*)(pte_t * , unsigned long),
1565 unsigned long );
1566 void set_data_memory_break(pte_t * , unsigned long);
1567 #endif
1568 /* resource map size dictated by pdir_size */
1569 res_size = sba_dev->ioc[i].pdir_size/sizeof(u64); /* entries */
1570 res_size >>= 3; /* convert bit count to byte count */
1571 DBG_INIT("%s() res_size 0x%x\n", __FUNCTION__, res_size);
1572
1573 sba_dev->ioc[i].res_size = res_size;
1574 sba_dev->ioc[i].res_map = (char *) __get_free_pages(GFP_KERNEL, get_order(res_size));
1575
1576 #ifdef CONFIG_DMB_TRAP
1577 iterate_pages( sba_dev->ioc[i].res_map, res_size,
1578 set_data_memory_break, 0);
1579 #endif
1580
1581 if (NULL == sba_dev->ioc[i].res_map)
1582 {
1583 panic(__FILE__ ":%s() could not allocate resource map\n", __FUNCTION__ );
1584 }
1585
1586 memset(sba_dev->ioc[i].res_map, 0, res_size);
1587 /* next available IOVP - circular search */
1588 sba_dev->ioc[i].res_hint = (unsigned long *)
1589 &(sba_dev->ioc[i].res_map[L1_CACHE_BYTES]);
1590
1591 #ifdef ASSERT_PDIR_SANITY
1592 /* Mark first bit busy - ie no IOVA 0 */
1593 sba_dev->ioc[i].res_map[0] = 0x80;
1594 sba_dev->ioc[i].pdir_base[0] = 0xeeffc0addbba0080ULL;
1595 #endif
1596
1597 #ifdef CONFIG_DMB_TRAP
1598 iterate_pages( sba_dev->ioc[i].res_map, res_size,
1599 set_data_memory_break, 0);
1600 iterate_pages( sba_dev->ioc[i].pdir_base, sba_dev->ioc[i].pdir_size,
1601 set_data_memory_break, 0);
1602 #endif
1603
1604 DBG_INIT("sba_common_init() %d res_map %x %p\n",
1605 i, res_size, sba_dev->ioc[i].res_map);
1606 }
1607
1608 sba_dev->sba_lock = SPIN_LOCK_UNLOCKED;
1609 }
1610
1611 #ifdef CONFIG_PROC_FS
1612 static int sba_proc_info(char *buf, char **start, off_t offset, int len)
1613 {
1614 struct sba_device *sba_dev = sba_list;
1615 /* FIXME: Multi-IOC support broken! */
1616 struct ioc *ioc = &sba_dev->ioc[0];
1617 int total_pages = (int) (ioc->res_size << 3); /* 8 bits per byte */
1618 unsigned long i = 0, avg = 0, min, max;
1619
1620 sprintf(buf, "%s rev %d.%d\n",
1621 parisc_getHWdescription(sba_dev->iodc->hw_type,
1622 sba_dev->iodc->hversion, sba_dev->iodc->sversion),
1623 (sba_dev->hw_rev & 0x7) + 1,
1624 (sba_dev->hw_rev & 0x18) >> 3
1625 );
1626 sprintf(buf, "%sIO PDIR size : %d bytes (%d entries)\n",
1627 buf,
1628 ((ioc->res_size << 3) * sizeof(u64)), /* 8 bits per byte */
1629 total_pages); /* 8 bits per byte */
1630
1631 sprintf(buf, "%sIO PDIR entries : %ld free %ld used (%d%%)\n", buf,
1632 total_pages - ioc->used_pages, ioc->used_pages,
1633 (int) (ioc->used_pages * 100 / total_pages));
1634
1635 sprintf(buf, "%sResource bitmap : %d bytes (%d pages)\n",
1636 buf, ioc->res_size, ioc->res_size << 3); /* 8 bits per byte */
1637
1638 min = max = ioc->avg_search[0];
1639 for (i = 0; i < SBA_SEARCH_SAMPLE; i++) {
1640 avg += ioc->avg_search[i];
1641 if (ioc->avg_search[i] > max) max = ioc->avg_search[i];
1642 if (ioc->avg_search[i] < min) min = ioc->avg_search[i];
1643 }
1644 avg /= SBA_SEARCH_SAMPLE;
1645 sprintf(buf, "%s Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles)\n",
1646 buf, min, avg, max);
1647
1648 sprintf(buf, "%spci_map_single(): %8ld calls %8ld pages (avg %d/1000)\n",
1649 buf, ioc->msingle_calls, ioc->msingle_pages,
1650 (int) ((ioc->msingle_pages * 1000)/ioc->msingle_calls));
1651
1652 /* KLUGE - unmap_sg calls unmap_single for each mapped page */
1653 min = ioc->usingle_calls - ioc->usg_calls;
1654 max = ioc->usingle_pages - ioc->usg_pages;
1655 sprintf(buf, "%spci_unmap_single: %8ld calls %8ld pages (avg %d/1000)\n",
1656 buf, min, max,
1657 (int) ((max * 1000)/min));
1658
1659 sprintf(buf, "%spci_map_sg() : %8ld calls %8ld pages (avg %d/1000)\n",
1660 buf, ioc->msg_calls, ioc->msg_pages,
1661 (int) ((ioc->msg_pages * 1000)/ioc->msg_calls));
1662
1663 sprintf(buf, "%spci_unmap_sg() : %8ld calls %8ld pages (avg %d/1000)\n",
1664 buf, ioc->usg_calls, ioc->usg_pages,
1665 (int) ((ioc->usg_pages * 1000)/ioc->usg_calls));
1666
1667 return strlen(buf);
1668 }
1669
1670 static int
1671 sba_resource_map(char *buf, char **start, off_t offset, int len)
1672 {
1673 struct sba_device *sba_dev = sba_list;
1674 struct ioc *ioc = &sba_dev->ioc[0];
1675 unsigned long *res_ptr = (unsigned long *)ioc->res_map;
1676 int i;
1677
1678 for(i = 0; i < (ioc->res_size / sizeof(unsigned long)); ++i, ++res_ptr) {
1679 if ((i & 7) == 0)
1680 strcat(buf,"\n ");
1681 sprintf(buf, "%s %08lx", buf, *res_ptr);
1682 }
1683 strcat(buf, "\n");
1684
1685 return strlen(buf);
1686 }
1687 #endif
1688
1689 /*
1690 ** Determine if lba should claim this chip (return 0) or not (return 1).
1691 ** If so, initialize the chip and tell other partners in crime they
1692 ** have work to do.
1693 */
1694 int
1695 sba_driver_callback(struct hp_device *d, struct pa_iodc_driver *dri)
1696 {
1697 struct sba_device *sba_dev;
1698 u32 func_class;
1699 int i;
1700
1701 if (IS_ASTRO(d)) {
1702 static char astro_rev[]="Astro ?.?";
1703
1704 /* Read HW Rev First */
1705 func_class = READ_REG32(d->hpa);
1706
1707 astro_rev[6] = '1' + (char) (func_class & 0x7);
1708 astro_rev[8] = '0' + (char) ((func_class & 0x18) >> 3);
1709 dri->version = astro_rev;
1710 } else {
1711 static char ike_rev[]="Ike rev ?";
1712
1713 /* Read HW Rev First */
1714 func_class = READ_REG32(d->hpa + SBA_FCLASS);
1715
1716 ike_rev[8] = '0' + (char) (func_class & 0xff);
1717 dri->version = ike_rev;
1718 }
1719
1720 printk("%s found %s at 0x%p\n", dri->name, dri->version, d->hpa);
1721
1722 sba_dev = kmalloc(sizeof(struct sba_device), GFP_KERNEL);
1723 if (NULL == sba_dev)
1724 {
1725 printk(MODULE_NAME " - couldn't alloc sba_device\n");
1726 return(1);
1727 }
1728 memset(sba_dev, 0, sizeof(struct sba_device));
1729 for(i=0; i<MAX_IOC; i++)
1730 spin_lock_init(&(sba_dev->ioc[i].res_lock));
1731
1732
1733 sba_dev->hw_rev = func_class;
1734 sba_dev->iodc = d;
1735 sba_dev->sba_hpa = d->hpa; /* faster access */
1736
1737 sba_get_pat_resources(sba_dev);
1738 sba_hw_init(sba_dev);
1739 sba_common_init(sba_dev);
1740
1741 hppa_dma_ops = &sba_ops;
1742
1743 #ifdef CONFIG_PROC_FS
1744 if (IS_ASTRO(d)) {
1745 create_proc_info_entry("Astro", 0, proc_runway_root, sba_proc_info);
1746 } else {
1747 create_proc_info_entry("Ike", 0, proc_runway_root, sba_proc_info);
1748 }
1749 create_proc_info_entry("bitmap", 0, proc_runway_root, sba_resource_map);
1750 #endif
1751 return 0;
1752 }
1753